Skip to content

Commit

Permalink
Fixing potential issues at runtime related with 3GPP Processing.
Browse files Browse the repository at this point in the history 
The Sftp3gppVTDXmlCollectionHandler is working fine (processing a full
3GPP XML in about 1 second).
  • Loading branch information
@agalue
agalue committed May 1, 2014
1 parent ef97cd3 commit a5978b3
Show file tree
Hide file tree
Showing 4 changed files with 34 additions and 54 deletions.
32 changes: 17 additions & 15 deletions ...s/xml/src/main/java/org/opennms/protocols/xml/collector/AbstractXmlCollectionHandler.java
View file
Expand Up @@ -183,20 +183,20 @@ public XmlCollectionSet collect(CollectionAgent agent, XmlDataCollection collect
LOG.debug("collect: starting source url '{}' collection", source.getUrl());
String urlStr = parseUrl(source.getUrl(), agent, collection.getXmlRrd().getStep());
LOG.debug("collect: parsed url for source url '{}'", source.getUrl());
Request request = parseRequest(source.getRequest(), agent);
Request request = parseRequest(source.getRequest(), agent, collection.getXmlRrd().getStep());
LOG.debug("collect: parsed request for source url '{}'", source.getUrl());
fillCollectionSet(urlStr, request, agent, collectionSet, source);
LOG.debug("collect: finished source url '{}' collection", source.getUrl());
}
collectionSet.setStatus(ServiceCollector.COLLECTION_SUCCEEDED);
DateTime endTime = new DateTime();
LOG.debug("collect: finished collection {}: duration: {} ms", collection.getName(), endTime.getMillis()-startTime.getMillis());
return collectionSet;
} catch (Exception e) {
collectionSet.setStatus(ServiceCollector.COLLECTION_FAILED);
DateTime endTime = new DateTime();
LOG.debug("collect: failed collection {}: duration: {} ms", collection.getName(), endTime.getMillis()-startTime.getMillis());
throw new CollectionException(e.getMessage(), e);
} finally {
String status = collectionSet.getStatus() == ServiceCollector.COLLECTION_SUCCEEDED ? "finished" : "failed";
DateTime endTime = new DateTime();
LOG.debug("collect: {} collection {}: duration: {} ms", status, collection.getName(), endTime.getMillis()-startTime.getMillis());
}
}

Expand Down Expand Up @@ -356,38 +356,38 @@ protected Date getTimeStamp(Document doc, XPath xpath, XmlGroup group) throws XP
*/
protected String parseUrl(final String unformattedUrl, final CollectionAgent agent, final Integer collectionStep) throws IllegalArgumentException {
final OnmsNode node = getNodeDao().get(agent.getNodeId());
String url = parseString("URL", unformattedUrl, node, agent.getHostAddress());
return url.replaceAll("[{]step[}]", collectionStep.toString());
return parseString("URL", unformattedUrl, node, agent.getHostAddress(), collectionStep);
}

/**
* Parses the request.
*
* @param unformattedRequest the unformatted request
* @param agent the agent
* @param collectionStep the collection step
* @return the request
* @throws IllegalArgumentException the illegal argument exception
*/
protected Request parseRequest(final Request unformattedRequest, final CollectionAgent agent) throws IllegalArgumentException {
protected Request parseRequest(final Request unformattedRequest, final CollectionAgent agent, final Integer collectionStep) throws IllegalArgumentException {
if (unformattedRequest == null)
return null;
final OnmsNode node = getNodeDao().get(agent.getNodeId());
final Request request = new Request();
for (Header header : unformattedRequest.getHeaders()) {
request.addHeader(header.getName(), parseString(header.getName(), header.getValue(), node, agent.getHostAddress()));
request.addHeader(header.getName(), parseString(header.getName(), header.getValue(), node, agent.getHostAddress(), collectionStep));
}
for (Parameter param : unformattedRequest.getParameters()) {
request.addParameter(param.getName(), parseString(param.getName(), param.getValue(), node, agent.getHostAddress()));
request.addParameter(param.getName(), parseString(param.getName(), param.getValue(), node, agent.getHostAddress(), collectionStep));
}
final Content cnt = unformattedRequest.getContent();
if (cnt != null)
request.setContent(new Content(cnt.getType(), parseString("Content", cnt.getData(), node, agent.getHostAddress())));
request.setContent(new Content(cnt.getType(), parseString("Content", cnt.getData(), node, agent.getHostAddress(), collectionStep)));
return request;
}

/**
* Parses the string.
*
*
* <p>Valid placeholders are:</p>
* <ul>
* <li><b>ipaddr</b>, The Node IP Address</li>
Expand All @@ -398,18 +398,20 @@ protected Request parseRequest(final Request unformattedRequest, final Collectio
* <li><b>foreignSource</b>, The Node Foreign Source</li>
* <li>Any asset property defined on the node.</li>
* </ul>
*
*
* @param reference the reference
* @param unformattedString the unformatted string
* @param node the node
* @param ipAddress the IP address
* @param collectionStep the collection step
* @return the string
* @throws IllegalArgumentException the illegal argument exception
*/
protected String parseString(final String reference, final String unformattedString, final OnmsNode node, final String ipAddress) throws IllegalArgumentException {
if (unformattedString == null)
protected String parseString(final String reference, final String unformattedString, final OnmsNode node, final String ipAddress, final Integer collectionStep) throws IllegalArgumentException {
if (unformattedString == null || node == null)
return null;
String formattedString = unformattedString.replaceAll("[{](?i)(ipAddr|ipAddress)[}]", ipAddress);
formattedString = formattedString.replaceAll("[{](?i)step[}]", collectionStep.toString());
formattedString = formattedString.replaceAll("[{](?i)nodeId[}]", node.getNodeId());
if (node.getLabel() != null)
formattedString = formattedString.replaceAll("[{](?i)nodeLabel[}]", node.getLabel());
Expand Down
25 changes: 7 additions & 18 deletions ...s/xml/src/main/java/org/opennms/protocols/xml/collector/Sftp3gppXmlCollectionHandler.java
View file
Expand Up @@ -36,6 +36,7 @@
import java.util.Map;

import org.apache.commons.io.IOUtils;
import org.joda.time.DateTime;
import org.opennms.netmgt.collection.api.AttributeGroupType;
import org.opennms.netmgt.collection.api.CollectionAgent;
import org.opennms.netmgt.collection.api.CollectionException;
Expand Down Expand Up @@ -74,14 +75,15 @@ public XmlCollectionSet collect(CollectionAgent agent, XmlDataCollection collect
collectionSet.setStatus(ServiceCollector.COLLECTION_UNKNOWN);

// TODO We could be careful when handling exceptions because parsing exceptions will be treated different from connection or retrieval exceptions
DateTime startTime = new DateTime();
try {
File resourceDir = new File(getRrdRepository().getRrdBaseDir(), Integer.toString(agent.getNodeId()));
for (XmlSource source : collection.getXmlSources()) {
if (!source.getUrl().startsWith(Sftp3gppUrlHandler.PROTOCOL)) {
throw new CollectionException("The 3GPP SFTP Collection Handler can only use the protocol " + Sftp3gppUrlHandler.PROTOCOL);
}
String urlStr = parseUrl(source.getUrl(), agent, collection.getXmlRrd().getStep());
Request request = parseRequest(source.getRequest(), agent);
Request request = parseRequest(source.getRequest(), agent, collection.getXmlRrd().getStep());
URL url = UrlFactory.getUrl(urlStr, request);
String lastFile = Sftp3gppUtils.getLastFilename(getServiceName(), resourceDir, url.getPath());
Sftp3gppUrlConnection connection = (Sftp3gppUrlConnection) url.openConnection();
Expand Down Expand Up @@ -120,6 +122,10 @@ public XmlCollectionSet collect(CollectionAgent agent, XmlDataCollection collect
} catch (Exception e) {
collectionSet.setStatus(ServiceCollector.COLLECTION_FAILED);
throw new CollectionException(e.getMessage(), e);
} finally {
String status = collectionSet.getStatus() == ServiceCollector.COLLECTION_SUCCEEDED ? "finished" : "failed";
DateTime endTime = new DateTime();
LOG.debug("collect: {} collection {}: duration: {} ms", status, collection.getName(), endTime.getMillis()-startTime.getMillis());
}
}

Expand All @@ -139,21 +145,4 @@ protected void processXmlResource(XmlCollectionResource resource, AttributeGroup
Sftp3gppUtils.processXmlResource(resource, attribGroupType);
}

/**
* Parses the URL.
*
* @param unformattedUrl the unformatted URL
* @param agent the agent
* @param collectionStep the collection step (in seconds)
* @param currentTimestamp the current timestamp
* @return the string
*/
protected String parseUrl(String unformattedUrl, CollectionAgent agent, Integer collectionStep, long currentTimestamp) throws IllegalArgumentException {
if (!unformattedUrl.startsWith(Sftp3gppUrlHandler.PROTOCOL)) {
throw new IllegalArgumentException("The 3GPP SFTP Collection Handler can only use the protocol " + Sftp3gppUrlHandler.PROTOCOL);
}
String baseUrl = parseUrl(unformattedUrl, agent, collectionStep);
return baseUrl + "&referenceTimestamp=" + currentTimestamp;
}

}
25 changes: 7 additions & 18 deletions ...s/xml/src/main/java/org/opennms/protocols/xml/vtdxml/Sftp3gppVTDXmlCollectionHandler.java
View file
Expand Up @@ -36,6 +36,7 @@
import java.util.Map;

import org.apache.commons.io.IOUtils;
import org.joda.time.DateTime;
import org.opennms.netmgt.collection.api.AttributeGroupType;
import org.opennms.netmgt.collection.api.CollectionAgent;
import org.opennms.netmgt.collection.api.CollectionException;
Expand Down Expand Up @@ -82,14 +83,15 @@ public XmlCollectionSet collect(CollectionAgent agent, XmlDataCollection collect
collectionSet.setStatus(ServiceCollector.COLLECTION_UNKNOWN);

// TODO We could be careful when handling exceptions because parsing exceptions will be treated different from connection or retrieval exceptions
DateTime startTime = new DateTime();
try {
File resourceDir = new File(getRrdRepository().getRrdBaseDir(), Integer.toString(agent.getNodeId()));
for (XmlSource source : collection.getXmlSources()) {
if (!source.getUrl().startsWith(Sftp3gppUrlHandler.PROTOCOL)) {
throw new CollectionException("The 3GPP SFTP Collection Handler can only use the protocol " + Sftp3gppUrlHandler.PROTOCOL);
}
String urlStr = parseUrl(source.getUrl(), agent, collection.getXmlRrd().getStep());
Request request = parseRequest(source.getRequest(), agent);
Request request = parseRequest(source.getRequest(), agent, collection.getXmlRrd().getStep());
URL url = UrlFactory.getUrl(urlStr, request);
String lastFile = Sftp3gppUtils.getLastFilename(getServiceName(), resourceDir, url.getPath());
Sftp3gppUrlConnection connection = (Sftp3gppUrlConnection) url.openConnection();
Expand Down Expand Up @@ -128,6 +130,10 @@ public XmlCollectionSet collect(CollectionAgent agent, XmlDataCollection collect
} catch (Exception e) {
collectionSet.setStatus(ServiceCollector.COLLECTION_FAILED);
throw new CollectionException(e.getMessage(), e);
} finally {
String status = collectionSet.getStatus() == ServiceCollector.COLLECTION_SUCCEEDED ? "finished" : "failed";
DateTime endTime = new DateTime();
LOG.debug("collect: {} collection {}: duration: {} ms", status, collection.getName(), endTime.getMillis()-startTime.getMillis());
}
}

Expand All @@ -147,21 +153,4 @@ protected void processXmlResource(XmlCollectionResource resource, AttributeGroup
Sftp3gppUtils.processXmlResource(resource, attribGroupType);
}

/**
* Parses the URL.
*
* @param unformattedUrl the unformatted URL
* @param agent the agent
* @param collectionStep the collection step (in seconds)
* @param currentTimestamp the current timestamp
* @return the string
*/
protected String parseUrl(String unformattedUrl, CollectionAgent agent, Integer collectionStep, long currentTimestamp) throws IllegalArgumentException {
if (!unformattedUrl.startsWith(Sftp3gppUrlHandler.PROTOCOL)) {
throw new IllegalArgumentException("The 3GPP SFTP Collection Handler can only use the protocol " + Sftp3gppUrlHandler.PROTOCOL);
}
String baseUrl = parseUrl(unformattedUrl, agent, collectionStep);
return baseUrl + "&referenceTimestamp=" + currentTimestamp;
}

}
6 changes: 3 additions & 3 deletions ...l/src/test/java/org/opennms/protocols/xml/collector/AbstractXmlCollectionHandlerTest.java
View file
Expand Up @@ -83,10 +83,10 @@ protected void fillCollectionSet(String urlString,
OnmsAssetRecord asset = new OnmsAssetRecord();
asset.setSerialNumber("1001");
node.setAssetRecord(asset);
String url = handler.parseString("URL", "http://{nodeLabel}/{ipAddress}/serial/{serialNumber}", node, "127.0.0.1");
Assert.assertEquals("http://mynode.local/127.0.0.1/serial/1001", url);
String url = handler.parseString("URL", "http://{nodeLabel}/{ipAddress}/serial/{serialNumber}/{step}", node, "127.0.0.1", 300);
Assert.assertEquals("http://mynode.local/127.0.0.1/serial/1001/300", url);
String multiline = "<data>\n <source label='{nodeLabel}'/>\n</data>";
String xml = handler.parseString("Content", multiline, node, "127.0.0.1");
String xml = handler.parseString("Content", multiline, node, "127.0.0.1", 300);
Assert.assertEquals("<data>\n <source label='mynode.local'/>\n</data>", xml);
}

Expand Down

0 comments on commit a5978b3

Please sign in to comment.