Skip to content

Commit

Permalink
Use JDOM in the XmlPublisherFilter to get XML character escaping.
Browse files Browse the repository at this point in the history
  • Loading branch information
chenson42 committed Nov 1, 2008
1 parent d31b7bc commit 44e77ad
Show file tree
Hide file tree
Showing 5 changed files with 193 additions and 51 deletions.
6 changes: 6 additions & 0 deletions symmetric/pom.xml
Expand Up @@ -487,6 +487,12 @@
<version>0.9</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.jdom</groupId>
<artifactId>jdom</artifactId>
<version>1.1</version>
<optional>true</optional>
</dependency>
<!-- Databases -->
<dependency>
<groupId>org.apache.derby</groupId>
Expand Down
6 changes: 6 additions & 0 deletions symmetric/src/changes/changes.xml
Expand Up @@ -5,6 +5,12 @@
<author email="chenson42@users.sourceforge.net">Chris Henson</author>
</properties>
<body>
<release version="1.4.2" date="2008-11-xx" description="Patch Release">
<action dev="chenson42" type="fix">
Use JDOM in XmlPublisherFilter to formatting the XML document. If you are upgrading and are currently doing the escaping characters for
XML, you'll probably want to remove that logic.
</action>
</release>
<release version="1.4.1" date="2008-10-21" description="Patch Release">
<action dev="chenson42" type="fix">
Batches were not being ordered correctly during the extraction when a channel was in error.
Expand Down
Expand Up @@ -21,6 +21,7 @@

import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
Expand All @@ -29,6 +30,11 @@
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.Namespace;
import org.jdom.output.Format;
import org.jdom.output.XMLOutputter;
import org.jumpmind.symmetric.load.IDataLoader;
import org.jumpmind.symmetric.load.IDataLoaderContext;
import org.jumpmind.symmetric.model.DataEventType;
Expand All @@ -37,11 +43,9 @@
/**
* This is an optional data loader filter/listener that is capable of
* translating table data to XML and publishing it to JMS for consumption by the
* enterprise.
* enterprise. It uses JDOM internally to create an XML representation of
* SymmetricDS data.
* </p>
* This class must be configured in the same context that SymmetricDS is running
* in. Simply inject the IDataLoaderService and it will register itself with the
* SymmetricDS engine.
*/
public class XmlPublisherFilter implements IPublisherFilter, INodeGroupExtensionPoint {

Expand All @@ -62,6 +66,19 @@ public class XmlPublisherFilter implements IPublisherFilter, INodeGroupExtension
private boolean loadDataInTargetDatabase = true;

private boolean autoRegister = true;

private Format xmlFormat;

private ITimeGenerator timeStringGenerator = new ITimeGenerator() {
public String getTime() {
return Long.toString(System.currentTimeMillis());
}
};

public XmlPublisherFilter() {
xmlFormat = Format.getCompactFormat();
xmlFormat.setOmitDeclaration(true);
}

public boolean isAutoRegister() {
return autoRegister;
Expand All @@ -73,7 +90,7 @@ public String[] getNodeGroupIdsToApplyTo() {

public boolean filterDelete(IDataLoaderContext ctx, String[] keys) {
if (tableNamesToPublishAsGroup == null || tableNamesToPublishAsGroup.contains(ctx.getTableName())) {
StringBuilder xml = getXmlFromCache(ctx, null, keys);
Element xml = getXmlFromCache(ctx, null, keys);
if (xml != null) {
toXmlElement(DataEventType.UPDATE, xml, ctx, null, keys);
}
Expand All @@ -83,7 +100,7 @@ public boolean filterDelete(IDataLoaderContext ctx, String[] keys) {

public boolean filterUpdate(IDataLoaderContext ctx, String[] data, String[] keys) {
if (tableNamesToPublishAsGroup == null || tableNamesToPublishAsGroup.contains(ctx.getTableName())) {
StringBuilder xml = getXmlFromCache(ctx, data, keys);
Element xml = getXmlFromCache(ctx, data, keys);
if (xml != null) {
toXmlElement(DataEventType.UPDATE, xml, ctx, data, keys);
}
Expand All @@ -93,35 +110,35 @@ public boolean filterUpdate(IDataLoaderContext ctx, String[] data, String[] keys

public boolean filterInsert(IDataLoaderContext ctx, String[] data) {
if (tableNamesToPublishAsGroup == null || tableNamesToPublishAsGroup.contains(ctx.getTableName())) {
StringBuilder xml = getXmlFromCache(ctx, data, null);
Element xml = getXmlFromCache(ctx, data, null);
if (xml != null) {
toXmlElement(DataEventType.INSERT, xml, ctx, data, null);
}
}
return loadDataInTargetDatabase;
}

private StringBuilder getXmlFromCache(IDataLoaderContext ctx, String[] data, String[] keys) {
StringBuilder xml = null;
Map<String, StringBuilder> ctxCache = getXmlCache(ctx);
private Element getXmlFromCache(IDataLoaderContext ctx, String[] data, String[] keys) {
Element xml = null;
Map<String, Element> ctxCache = getXmlCache(ctx);
String txId = toXmlGroupId(ctx, data, keys);
if (txId != null) {
xml = ctxCache.get(txId);
if (xml == null) {
xml = new StringBuilder();
xml.append("<");
xml.append(xmlTagNameToUseForGroup);
xml.append(" xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' id='");
xml.append(txId);
xml.append("' ");
xml = new Element(xmlTagNameToUseForGroup);
xml.addNamespaceDeclaration(getXmlNamespace());
xml.setAttribute("id", txId);
addFormattedExtraGroupAttributes(ctx, xml);
xml.append(">");
ctxCache.put(txId, xml);
}
}
return xml;
}

private final static Namespace getXmlNamespace() {
return Namespace.getNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");
}

/**
* Give the opportunity for the user of this publisher to add in additional
* attributes. The default implementation adds in the nodeId from the
Expand All @@ -131,20 +148,19 @@ private StringBuilder getXmlFromCache(IDataLoaderContext ctx, String[] data, Str
* @param xml
* append XML attributes to this buffer
*/
protected void addFormattedExtraGroupAttributes(IDataLoaderContext ctx, StringBuilder xml) {
xml.append("nodeid='");
xml.append(ctx.getNodeId());
xml.append("' time='");
xml.append(System.currentTimeMillis());
xml.append("'");
protected void addFormattedExtraGroupAttributes(IDataLoaderContext ctx, Element xml) {
xml.setAttribute("nodeid", ctx.getNodeId());
if (timeStringGenerator != null) {
xml.setAttribute("time", timeStringGenerator.getTime());
}
}

@SuppressWarnings("unchecked")
protected Map<String, StringBuilder> getXmlCache(IDataLoaderContext ctx) {
protected Map<String, Element> getXmlCache(IDataLoaderContext ctx) {
Map<String, Object> cache = ctx.getContextCache();
Map<String, StringBuilder> xmlCache = (Map<String, StringBuilder>) cache.get(XML_CACHE);
Map<String, Element> xmlCache = (Map<String, Element>) cache.get(XML_CACHE);
if (xmlCache == null) {
xmlCache = new HashMap<String, StringBuilder>();
xmlCache = new HashMap<String, Element>();
cache.put(XML_CACHE, xmlCache);
}
return xmlCache;
Expand All @@ -157,12 +173,11 @@ protected boolean doesXmlExistToPublish(IDataLoaderContext ctx) {
return xmlCache != null && xmlCache.size() > 0;
}

private void toXmlElement(DataEventType dml, StringBuilder xml, IDataLoaderContext ctx, String[] data, String[] keys) {
xml.append("<row entity='");
xml.append(ctx.getTableName());
xml.append("' dml='");
xml.append(dml.getCode());
xml.append("'>");
private void toXmlElement(DataEventType dml, Element xml, IDataLoaderContext ctx, String[] data, String[] keys) {
Element row = new Element("row");
xml.addContent(row);
row.setAttribute("entity", ctx.getTableName());
row.setAttribute("dml", dml.getCode());

String[] colNames = null;

Expand All @@ -175,19 +190,15 @@ private void toXmlElement(DataEventType dml, StringBuilder xml, IDataLoaderConte

for (int i = 0; i < data.length; i++) {
String col = colNames[i];
xml.append("<data key='");
xml.append(col);
xml.append("'");

if (data[i] == null) {
xml.append(" xsi:nil='true'/>");
Element dataElement = new Element("data");
row.addContent(dataElement);
dataElement.setAttribute("key", col);
if (data[i] != null) {
dataElement.setText(data[i]);
} else {
xml.append(">");
xml.append(data[i]);
xml.append("</data>");
dataElement.setAttribute("nil", "true", getXmlNamespace());
}
}
xml.append("</row>");
}

private String toXmlGroupId(IDataLoaderContext ctx, String[] data, String[] keys) {
Expand Down Expand Up @@ -222,25 +233,24 @@ private String toXmlGroupId(IDataLoaderContext ctx, String[] data, String[] keys
if (id.length() > 0) {
return id.toString().replaceAll("-", "");
}
} else {
logger.warn("You did not specify 'groupByColumnNames'. We cannot find any matches in the data to publish as XML if you don't. You might as well turn off this filter!");
}
return null;
}

private void finalizeXmlAndPublish(IDataLoaderContext ctx) {
Map<String, StringBuilder> ctxCache = getXmlCache(ctx);
Collection<StringBuilder> buffers = ctxCache.values();
for (Iterator<StringBuilder> iterator = buffers.iterator(); iterator.hasNext();) {
StringBuilder xml = iterator.next();
xml.append("</");
xml.append(xmlTagNameToUseForGroup);
xml.append(">");
Map<String, Element> ctxCache = getXmlCache(ctx);
Collection<Element> buffers = ctxCache.values();
for (Iterator<Element> iterator = buffers.iterator(); iterator.hasNext();) {
String xml = new XMLOutputter(xmlFormat).outputString(new Document(iterator.next()));
if (logger.isDebugEnabled()) {
logger.debug("Sending XML to IPublisher -> " + xml);
}
iterator.remove();
publisher.publish(ctx, xml.toString());
publisher.publish(ctx, xml.toString());
}

}

public void batchComplete(IDataLoader loader, IncomingBatchHistory hist) {
Expand All @@ -253,11 +263,20 @@ public void batchComplete(IDataLoader loader, IncomingBatchHistory hist) {
public void setTableNamesToPublishAsGroup(Set<String> tableNamesToPublishAsGroup) {
this.tableNamesToPublishAsGroup = tableNamesToPublishAsGroup;
}

public void setTableNameToPublish(String tableName) {
this.tableNamesToPublishAsGroup = new HashSet<String>(1);
this.tableNamesToPublishAsGroup.add(tableName);
}

public void setXmlTagNameToUseForGroup(String xmlTagNameToUseForGroup) {
this.xmlTagNameToUseForGroup = xmlTagNameToUseForGroup;
}

/**
* This attribute is required. It needs to identify the columns that will be used to key on
* rows in the specified tables that need to be grouped together in an 'XML batch.'
*/
public void setGroupByColumnNames(List<String> groupByColumnNames) {
this.groupByColumnNames = groupByColumnNames;
}
Expand All @@ -278,4 +297,19 @@ public void setAutoRegister(boolean autoRegister) {
this.autoRegister = autoRegister;
}

interface ITimeGenerator {
public String getTime();
}

/**
* Used to populate the time attribute of an XML message.
*/
public void setTimeStringGenerator(ITimeGenerator timeStringGenerator) {
this.timeStringGenerator = timeStringGenerator;
}

public void setXmlFormat(Format xmlFormat) {
this.xmlFormat = xmlFormat;
}

}
@@ -0,0 +1,88 @@
package org.jumpmind.symmetric.ext;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;

import org.jumpmind.symmetric.load.DataLoaderContext;
import org.jumpmind.symmetric.load.IDataLoaderContext;
import org.jumpmind.symmetric.load.TableTemplate;
import org.jumpmind.symmetric.load.csv.CsvLoader;
import org.jumpmind.symmetric.test.AbstractDatabaseTest;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

public class XmlPublisherFilterTest extends AbstractDatabaseTest {

private static final String TABLE_TEST = "TEST_XML_PUBLISHER";

private static final String TEST_SIMPLE_TRANSFORM_RESULTS = "<batch xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" id=\"12\" nodeid=\"54321\" time=\"test\"><row entity=\"TEST_XML_PUBLISHER\" dml=\"I\"><data key=\"ID1\">1</data><data key=\"ID2\">2</data><data key=\"DATA1\">test embedding an &amp;</data><data key=\"DATA2\">3</data><data key=\"DATA3\" xsi:nil=\"true\" /></row></batch>";

private DataLoaderContext ctx;

public XmlPublisherFilterTest() throws Exception {
super();
}

public XmlPublisherFilterTest(String dbName) {
super(dbName);
}

@Before
public void setUp() {
ctx = new DataLoaderContext();
ctx.setNodeId("54321");
ctx.setTableName(TABLE_TEST);
ctx.setTableTemplate(new TableTemplate(getJdbcTemplate(), getDbDialect(), TABLE_TEST, null, false));
ctx.setColumnNames(new String[] { "ID1", "ID2", "DATA1", "DATA2", "DATA3" });

}

@Test
public void testSimpleTransform() {
XmlPublisherFilter filter = new XmlPublisherFilter();
filter.setTimeStringGenerator(new XmlPublisherFilter.ITimeGenerator() {
public String getTime() {
return "test";
}
});
HashSet<String> tableNames = new HashSet<String>();
tableNames.add(TABLE_TEST);
filter.setTableNamesToPublishAsGroup(tableNames);
List<String> columns = new ArrayList<String>();
columns.add("ID1");
columns.add("ID2");
filter.setGroupByColumnNames(columns);
Output output = new Output();
filter.setPublisher(output);

String[][] data = { { "1", "1", "The Angry Brown", "3", "2008-10-24 00:00:00.0" },
{ "1", "2", "test embedding an &", "3", null } };
for (String[] strings : data) {
filter.filterInsert(ctx, strings);
filter.batchComplete(new CsvLoader() {
@Override
public IDataLoaderContext getContext() {
return ctx;
}
}, null);
}

Assert.assertEquals(TEST_SIMPLE_TRANSFORM_RESULTS.trim(), output.toString().trim());

}

class Output implements IPublisher {
private String output;

public void publish(IDataLoaderContext context, String text) {
this.output = text;
}

@Override
public String toString() {
return output;
}
}
}
8 changes: 8 additions & 0 deletions symmetric/src/test/resources/test-tables-ddl.xml
Expand Up @@ -98,5 +98,13 @@
<column name="Mixed_Case_Id" type="INTEGER" primaryKey="true" required="true" />
<column name="Name" type="VARCHAR" size="50" required="true" />
</table>

<table name="TEST_XML_PUBLISHER">
<column name="ID1" type="INTEGER" primaryKey="true" required="true" />
<column name="ID2" type="INTEGER" required="true" />
<column name="DATA1" type="VARCHAR" size="10" />
<column name="DATA2" type="INTEGER" />
<column name="DATA3" type="TIMESTAMP" />
</table>

</database>

0 comments on commit 44e77ad

Please sign in to comment.