srm: Fix ConcurrentModificationException in Axis

Adds an aspect to protect against the following error: 19 Mar 2015 08:29:54 (SRM-bunsen) [] Likely bug detected. Please report this to support@dcache.org. java.util.ConcurrentModificationException: null at java.util.ArrayList.sort(ArrayList.java:1456) ~[na:1.8.0_31] at java.util.Collections.sort(Collections.java:175) ~[na:1.8.0_31] at org.apache.axis.description.JavaServiceDesc.getOperationsByQName(JavaServiceDesc.java:527) ~[axis-1.4.jar:na] at org.apache.axis.MessageContext.getPossibleOperationsByQName(MessageContext.java:243) ~[axis-1.4.jar:na] at org.apache.axis.message.BodyBuilder.onStartChild(BodyBuilder.java:132) ~[axis-1.4.jar:na] at org.apache.axis.encoding.DeserializationContext.startElement(DeserializationContext.java:1035) ~[axis-1.4.jar:na] at org.apache.xerces.parsers.AbstractSAXParser.startElement(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.impl.XMLNSDocumentScannerImpl.scanStartElement(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.impl.XMLDocumentFragmentScannerImpl$FragmentContentDispatcher.dispatch(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.impl.XMLDocumentFragmentScannerImpl.scanDocument(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.parsers.XMLParser.parse(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.parsers.AbstractSAXParser.parse(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.jaxp.SAXParserImpl$JAXPSAXParser.parse(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.jaxp.SAXParserImpl.parse(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.axis.encoding.DeserializationContext.parse(DeserializationContext.java:227) ~[axis-1.4.jar:na] at org.apache.axis.SOAPPart.getAsSOAPEnvelope(SOAPPart.java:696) ~[axis-1.4.jar:na] at org.apache.axis.Message.getSOAPEnvelope(Message.java:435) ~[axis-1.4.jar:na] at org.apache.axis.server.AxisServer.initSOAPConstants(AxisServer.java:345) ~[axis-1.4.jar:na] at org.apache.axis.server.AxisServer.invoke(AxisServer.java:279) ~[axis-1.4.jar:na] at org.apache.axis.transport.http.AxisServlet.doPost(AxisServlet.java:699) [axis-1.4.jar:na] at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) [javax.servlet-api-3.1.0.jar:3.1.0] at org.apache.axis.transport.http.AxisServletBase.service(AxisServletBase.java:327) [axis-1.4.jar:na] at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) [javax.servlet-api-3.1.0.jar:3.1.0] at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:808) [jetty-servlet-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:587) [jetty-servlet-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577) [jetty-security-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) [jetty-servlet-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.Server.handle(Server.java:497) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540) [jetty-io-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) [jetty-util-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) [jetty-util-9.2.8.v20150217.jar:9.2.8.v20150217] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_31] Axis fails to protect a shared data structure. The aspect adds synchronization points around the public methods of the JavaServiceDesc class. The patch adds an inactive debugging aspect for logging ConcurrentModificationExceptions. That aspect was used to locate the bug. Target: trunk Require-notes: yes Require-book: no Request: 2.12 Request: 2.11 Request: 2.10 Acked-by: Paul Millar <paul.millar@desy.de> Patch: https://rb.dcache.org/r/7987/ (cherry picked from commit 8a8b3dd)
dCache · Mar 23, 2015 · 431e4e0 · 431e4e0
1 parent e0c06ce
commit 431e4e0
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 0 deletions.
diff --git a/...les/dcache/src/main/aspect/org/dcache/util/aspects/ConcurrentModificationLoggingAspect.aj b/...les/dcache/src/main/aspect/org/dcache/util/aspects/ConcurrentModificationLoggingAspect.aj
@@ -0,0 +1,20 @@
+package org.dcache.util.aspects;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ConcurrentModificationException;
+
+/**
+ * Aspect that will log whenever a ConcurrentModificationException is thrown.
+ */
+public aspect ConcurrentModificationLoggingAspect
+{
+    private static final Logger LOGGER = LoggerFactory.getLogger(ConcurrentModificationLoggingAspect.class);
+
+    before(Throwable e): (handler(java.lang.Throwable) || handler(java.lang.Exception) || handler(java.lang.RuntimeException) || handler(ConcurrentModificationException)) && args(e) {
+        if (e instanceof ConcurrentModificationException) {
+            LOGGER.warn("Likely bug detected. Please report this to support@dcache.org.", e);
+        }
+    }
+}
diff --git a/...es/dcache/src/main/aspect/org/dcache/util/aspects/JavaServiceDescSynchronizationAspect.aj b/...es/dcache/src/main/aspect/org/dcache/util/aspects/JavaServiceDescSynchronizationAspect.aj
@@ -0,0 +1,19 @@
+package org.dcache.util.aspects;
+
+import org.apache.axis.description.JavaServiceDesc;
+
+/**
+ * Axis fails to synchronize access to internal collections in JavaServiceDesc. Instances
+ * of that class are shared by concurrent requests and access thus has to be synchronized.
+ */
+public aspect JavaServiceDescSynchronizationAspect
+{
+    pointcut publicMethodExecution(JavaServiceDesc o) :
+            execution(public * JavaServiceDesc.*(..)) && this(o);
+
+    Object around(JavaServiceDesc o) : publicMethodExecution(o) {
+        synchronized (o) {
+            return proceed(o);
+        }
+    }
+}
diff --git a/modules/dcache/src/main/resources/META-INF/aop.xml b/modules/dcache/src/main/resources/META-INF/aop.xml
@@ -5,10 +5,12 @@
         <include within="diskCacheV111.services.space.*"/>
         <include within="org.dcache.pinmanager.*"/>
         <include within="org.dcache.util.aspects.*"/>
+        <include within="org.apache.axis.description.JavaServiceDesc"/>
     </weaver>
 
     <aspects>
         <exclude within="org.springframework.transaction.aspectj.AnnotationTransactionAspect"/>
+        <aspect name="org.dcache.util.aspects.JavaServiceDescSynchronizationAspect"/>
         <aspect name="org.dcache.util.aspects.PerInstanceAnnotationTransactionAspect"/>
     </aspects>
 </aspectj>