Add JSON API for dCache

The patch provides a response engine for the httpd service. The 
response engine translates users queries to UniversalSpringCell
queries. The query uses reflection to access bean properties. 
It uses a custom serialization scheme to provide a tree of 
public properties (public getters). The response engine 
translates the query response to JSON.

The API is read-only. It is exposed in the default httpd setup
under the api path:

set alias api class org.dcache.services.httpd.probe.ProbeResponseEngine

The API is accessed through HTTP, eg http://localhost:2288/api/PoolManager
or http://localhost:2288/api/PoolManager/psu.activePools.

In contrast to the info service, API requests trigger queries
to the actual service. Thus the information is guaranteed to be
up to date. The downside is that frequent probes would incure
overhead. The API is not guaranteed to be stable at this point (ie
we may change it in between releases). The goal is that any information
that can extracted through the ssh interface is also alvailable through
the API.

Target: trunk
Require-notes: yes
Require-book: yes

modules/dcache/pom.xml

@@ -276,6 +276,11 @@
         <artifactId>jython</artifactId>
         <scope>test</scope>
     </dependency>
+      <dependency>
+          <groupId>com.google.code.gson</groupId>
+          <artifactId>gson</artifactId>
+      </dependency>
+
   </dependencies>
 
   <build>

modules/dcache/src/main/java/org/dcache/cells/UniversalSpringCell.java

@@ -4,12 +4,19 @@
 import com.google.common.base.Strings;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Multimaps;
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanInstantiationException;
 import org.springframework.beans.BeanWrapper;
 import org.springframework.beans.BeanWrapperImpl;
 import org.springframework.beans.BeansException;
+import org.springframework.beans.InvalidPropertyException;
+import org.springframework.beans.PropertyAccessException;
 import org.springframework.beans.factory.BeanCreationException;
 import org.springframework.beans.factory.NoSuchBeanDefinitionException;
 import org.springframework.beans.factory.config.BeanDefinition;
@@ -32,6 +39,8 @@
 import java.io.PrintWriter;
 import java.io.Serializable;
 import java.io.StringWriter;
+import java.lang.reflect.Array;
+import java.util.ArrayDeque;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -40,12 +49,15 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Queue;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 
+import diskCacheV111.util.CacheException;
+
 import dmg.cells.nucleus.CellInfo;
 import dmg.cells.nucleus.CellMessage;
 import dmg.cells.nucleus.CellPath;
@@ -57,6 +69,9 @@
 import dmg.util.CommandThrowableException;
 
 import org.dcache.util.ClassNameComparator;
+import org.dcache.vehicles.BeanQueryAllPropertiesMessage;
+import org.dcache.vehicles.BeanQueryMessage;
+import org.dcache.vehicles.BeanQuerySinglePropertyMessage;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
@@ -85,6 +100,9 @@ public class UniversalSpringCell
 {
     private final static long WAIT_FOR_FILE_SLEEP = 30000;
 
+    private final static Logger _log = LoggerFactory
+            .getLogger(UniversalSpringCell.class);
+
     /**
      * Environment map this cell was instantiated in.
      */
@@ -757,6 +775,131 @@ protected String getMessageName(Class<?> c)
         }
     }
 
+    public BeanQueryMessage messageArrived(BeanQueryAllPropertiesMessage message)
+            throws CacheException
+    {
+        Map<String,Object> beans = Maps.newHashMap();
+        for (String name : getBeanNames()) {
+            beans.put(name, getBean(name));
+        }
+        message.setResult(serialize(beans));
+        return message;
+    }
+
+    public BeanQueryMessage messageArrived(BeanQuerySinglePropertyMessage message)
+            throws CacheException
+    {
+        Object o = getBeanProperty(message.getPropertyName());
+        if (o == null) {
+            throw new CacheException("No such property");
+        }
+        message.setResult(serialize(o));
+        return message;
+    }
+
+    private final static Set<Class<?>> PRIMITIVES =
+            Sets.<Class<?>>newHashSet(Byte.class, Byte.TYPE, Short.class, Short.TYPE,
+                    Integer.class, Integer.TYPE, Long.class, Long.TYPE,
+                    Float.class, Float.TYPE, Double.class, Double.TYPE,
+                    Character.class, Character.TYPE, Boolean.class,
+                    Boolean.TYPE, String.class);
+    private final static Set<Class<?>> TERMINALS =
+            Sets.<Class<?>>newHashSet(Class.class);
+
+    private Object serialize(Set<Object> prune, Queue<Map.Entry<String,Object>> queue, Object o)
+    {
+        if (o == null || PRIMITIVES.contains(o.getClass())) {
+            return o;
+        } else if (TERMINALS.contains(o.getClass())) {
+            return o.toString();
+        } else if (o.getClass().isEnum()) {
+            return o;
+        } else if (o.getClass().isArray()) {
+            int len = Array.getLength(o);
+            List<Object> values = Lists.newArrayListWithCapacity(len);
+            for (int i = 0; i < len; i++) {
+                values.add(serialize(prune, queue, Array.get(o, i)));
+            }
+            return values;
+        } else if (o instanceof Map) {
+            Map<?,?> map = (Map<?,?>) o;
+            Map<String,Object> values = Maps.newHashMapWithExpectedSize(map.size());
+            for (Map.Entry<?,?> e: map.entrySet()) {
+                values.put(String.valueOf(e.getKey()), serialize(prune, queue, e
+                        .getValue()));
+            }
+            return values;
+        } else if (o instanceof Set) {
+            Collection<?> collection = (Collection<?>) o;
+            Set<Object> values = Sets.newHashSetWithExpectedSize(collection.size());
+            for (Object entry: collection) {
+                values.add(serialize(prune, queue, entry));
+            }
+            return values;
+        } else if (o instanceof Collection) {
+            Collection<?> collection = (Collection<?>) o;
+            List<Object> values = Lists.newArrayListWithCapacity(collection.size());
+            for (Object entry: collection) {
+                values.add(serialize(prune, queue, entry));
+            }
+            return values;
+        } else if (o instanceof Iterable) {
+            Iterable<?> collection = (Iterable<?>) o;
+            List<Object> values = Lists.newArrayList();
+            for (Object entry: collection) {
+                values.add(serialize(prune, queue, entry));
+            }
+            return values;
+        } else if (prune.contains(o)) {
+            return o.toString();
+        } else {
+            prune.add(o);
+
+            Map<String,Object> values = Maps.newHashMap();
+            BeanWrapper bean = new BeanWrapperImpl(o);
+            for (PropertyDescriptor p: bean.getPropertyDescriptors()) {
+                if (!p.isHidden()) {
+                    String property = p.getName();
+                    if (bean.isReadableProperty(property)) {
+                        try {
+                            values.put(property, bean.getPropertyValue(property));
+                        } catch (InvalidPropertyException | PropertyAccessException e) {
+                            _log.debug("Failed to read {} of object of class {}: {}",
+                                    new Object[] { property, o.getClass(), e.getMessage() });
+                        }
+                    }
+                }
+            }
+            if (values.isEmpty()) {
+                return o.toString();
+            }
+            queue.addAll(values.entrySet());
+            return values;
+        }
+    }
+
+    /**
+     * Breadth-first serialisation.
+     *
+     * Prunes the object tree to produce a tree even if o is a DAG or
+     * contains cycles. Using a breadth-first search tends to produce
+     * friendlier results when the object graph is pruned.
+     */
+    private Object serialize(Object o)
+    {
+        Set<Object> prune = Sets.newHashSet();
+        Queue<Map.Entry<String,Object>> queue = new ArrayDeque<>();
+        Object result = serialize(prune, queue, o);
+
+        Map.Entry<String,Object> entry;
+        while ((entry = queue.poll()) != null) {
+            entry.setValue(serialize(prune, queue, entry.getValue()));
+        }
+
+        return result;
+    }
+
+
     /**
      * Registers an info provider. Info providers contribute to the
      * result of the <code>getInfo</code> method.

pom.xml

@@ -687,6 +687,13 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+
+            <dependency>
+                <groupId>com.google.code.gson</groupId>
+                <artifactId>gson</artifactId>
+                <version>2.2.2</version>
+            </dependency>
+
             <dependency>
                 <groupId>org.freehep</groupId>
                 <artifactId>freehep-graphics2d</artifactId>

skel/share/services/httpd.batch

@@ -108,6 +108,7 @@ define context ${cell.name}Setup endDefine
    set alias billing class diskCacheV111.cells.HttpBillingEngine
    set alias flushManager class diskCacheV111.hsmControl.flush.HttpHsmFlushMgrEngineV1 mgr=hfc css=default
    set alias pools class diskCacheV111.services.web.PoolInfoObserverEngineV2 showPoolGroupUsage=true
+   set alias api class org.dcache.services.httpd.probe.ProbeResponseEngine
    ${set_alias_statistics}
    set alias info class org.dcache.services.info.InfoHttpEngine
    exec context webadmin.exe