From 431e4e0099fc66d9db62f416d3dc03d4811069d7 Mon Sep 17 00:00:00 2001 From: Gerd Behrmann Date: Wed, 18 Mar 2015 22:41:09 +0100 Subject: [PATCH] srm: Fix ConcurrentModificationException in Axis Adds an aspect to protect against the following error: 19 Mar 2015 08:29:54 (SRM-bunsen) [] Likely bug detected. Please report this to support@dcache.org. java.util.ConcurrentModificationException: null at java.util.ArrayList.sort(ArrayList.java:1456) ~[na:1.8.0_31] at java.util.Collections.sort(Collections.java:175) ~[na:1.8.0_31] at org.apache.axis.description.JavaServiceDesc.getOperationsByQName(JavaServiceDesc.java:527) ~[axis-1.4.jar:na] at org.apache.axis.MessageContext.getPossibleOperationsByQName(MessageContext.java:243) ~[axis-1.4.jar:na] at org.apache.axis.message.BodyBuilder.onStartChild(BodyBuilder.java:132) ~[axis-1.4.jar:na] at org.apache.axis.encoding.DeserializationContext.startElement(DeserializationContext.java:1035) ~[axis-1.4.jar:na] at org.apache.xerces.parsers.AbstractSAXParser.startElement(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.impl.XMLNSDocumentScannerImpl.scanStartElement(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.impl.XMLDocumentFragmentScannerImpl$FragmentContentDispatcher.dispatch(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.impl.XMLDocumentFragmentScannerImpl.scanDocument(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.parsers.XMLParser.parse(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.parsers.AbstractSAXParser.parse(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.jaxp.SAXParserImpl$JAXPSAXParser.parse(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.xerces.jaxp.SAXParserImpl.parse(Unknown Source) ~[xercesImpl-2.11.0.jar:na] at org.apache.axis.encoding.DeserializationContext.parse(DeserializationContext.java:227) ~[axis-1.4.jar:na] at org.apache.axis.SOAPPart.getAsSOAPEnvelope(SOAPPart.java:696) ~[axis-1.4.jar:na] at org.apache.axis.Message.getSOAPEnvelope(Message.java:435) ~[axis-1.4.jar:na] at org.apache.axis.server.AxisServer.initSOAPConstants(AxisServer.java:345) ~[axis-1.4.jar:na] at org.apache.axis.server.AxisServer.invoke(AxisServer.java:279) ~[axis-1.4.jar:na] at org.apache.axis.transport.http.AxisServlet.doPost(AxisServlet.java:699) [axis-1.4.jar:na] at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) [javax.servlet-api-3.1.0.jar:3.1.0] at org.apache.axis.transport.http.AxisServletBase.service(AxisServletBase.java:327) [axis-1.4.jar:na] at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) [javax.servlet-api-3.1.0.jar:3.1.0] at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:808) [jetty-servlet-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:587) [jetty-servlet-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:577) [jetty-security-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1127) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) [jetty-servlet-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1061) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.Server.handle(Server.java:497) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:257) [jetty-server-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540) [jetty-io-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635) [jetty-util-9.2.8.v20150217.jar:9.2.8.v20150217] at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555) [jetty-util-9.2.8.v20150217.jar:9.2.8.v20150217] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_31] Axis fails to protect a shared data structure. The aspect adds synchronization points around the public methods of the JavaServiceDesc class. The patch adds an inactive debugging aspect for logging ConcurrentModificationExceptions. That aspect was used to locate the bug. Target: trunk Require-notes: yes Require-book: no Request: 2.12 Request: 2.11 Request: 2.10 Acked-by: Paul Millar Patch: https://rb.dcache.org/r/7987/ (cherry picked from commit 8a8b3ddfbedcb6cf3b9c27cc56b80b4a02a8dc40) --- .../ConcurrentModificationLoggingAspect.aj | 20 +++++++++++++++++++ .../JavaServiceDescSynchronizationAspect.aj | 19 ++++++++++++++++++ .../src/main/resources/META-INF/aop.xml | 2 ++ 3 files changed, 41 insertions(+) create mode 100644 modules/dcache/src/main/aspect/org/dcache/util/aspects/ConcurrentModificationLoggingAspect.aj create mode 100644 modules/dcache/src/main/aspect/org/dcache/util/aspects/JavaServiceDescSynchronizationAspect.aj diff --git a/modules/dcache/src/main/aspect/org/dcache/util/aspects/ConcurrentModificationLoggingAspect.aj b/modules/dcache/src/main/aspect/org/dcache/util/aspects/ConcurrentModificationLoggingAspect.aj new file mode 100644 index 00000000000..2e0976e6890 --- /dev/null +++ b/modules/dcache/src/main/aspect/org/dcache/util/aspects/ConcurrentModificationLoggingAspect.aj @@ -0,0 +1,20 @@ +package org.dcache.util.aspects; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ConcurrentModificationException; + +/** + * Aspect that will log whenever a ConcurrentModificationException is thrown. + */ +public aspect ConcurrentModificationLoggingAspect +{ + private static final Logger LOGGER = LoggerFactory.getLogger(ConcurrentModificationLoggingAspect.class); + + before(Throwable e): (handler(java.lang.Throwable) || handler(java.lang.Exception) || handler(java.lang.RuntimeException) || handler(ConcurrentModificationException)) && args(e) { + if (e instanceof ConcurrentModificationException) { + LOGGER.warn("Likely bug detected. Please report this to support@dcache.org.", e); + } + } +} diff --git a/modules/dcache/src/main/aspect/org/dcache/util/aspects/JavaServiceDescSynchronizationAspect.aj b/modules/dcache/src/main/aspect/org/dcache/util/aspects/JavaServiceDescSynchronizationAspect.aj new file mode 100644 index 00000000000..317062c0261 --- /dev/null +++ b/modules/dcache/src/main/aspect/org/dcache/util/aspects/JavaServiceDescSynchronizationAspect.aj @@ -0,0 +1,19 @@ +package org.dcache.util.aspects; + +import org.apache.axis.description.JavaServiceDesc; + +/** + * Axis fails to synchronize access to internal collections in JavaServiceDesc. Instances + * of that class are shared by concurrent requests and access thus has to be synchronized. + */ +public aspect JavaServiceDescSynchronizationAspect +{ + pointcut publicMethodExecution(JavaServiceDesc o) : + execution(public * JavaServiceDesc.*(..)) && this(o); + + Object around(JavaServiceDesc o) : publicMethodExecution(o) { + synchronized (o) { + return proceed(o); + } + } +} diff --git a/modules/dcache/src/main/resources/META-INF/aop.xml b/modules/dcache/src/main/resources/META-INF/aop.xml index e834c23b30d..e616531da3f 100644 --- a/modules/dcache/src/main/resources/META-INF/aop.xml +++ b/modules/dcache/src/main/resources/META-INF/aop.xml @@ -5,10 +5,12 @@ + +