Add services providing statistics about search and catalog content (r…

…eturn XML response). Enable stats by default.

core/src/main/java/org/fao/geonet/util/JODAISODate.java

@@ -113,7 +113,7 @@ public static String parseISODateTimes(String input1,String input2)
 		return odt;
 	}
 
-	private static DateTime parseBasicOrFullDateTime(String input1) throws Exception {
+	public static DateTime parseBasicOrFullDateTime(String input1) throws Exception {
 		DateTimeFormatter bd = ISODateTimeFormat.basicDate();
 		DateTimeFormatter bt = ISODateTimeFormat.basicTime();
 		DateTimeFormatter bdt = ISODateTimeFormat.basicDateTime();

services/src/main/java/org/fao/geonet/services/statistics/ContentStatistics.java

@@ -0,0 +1,77 @@
+//=============================================================================
+//===   Copyright (C) 2001-2013 Food and Agriculture Organization of the
+//===   United Nations (FAO-UN), United Nations World Food Programme (WFP)
+//===   and United Nations Environment Programme (UNEP)
+//===
+//===   This program is free software; you can redistribute it and/or modify
+//===   it under the terms of the GNU General Public License as published by
+//===   the Free Software Foundation; either version 2 of the License, or (at
+//===   your option) any later version.
+//===
+//===   This program is distributed in the hope that it will be useful, but
+//===   WITHOUT ANY WARRANTY; without even the implied warranty of
+//===   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//===   General Public License for more details.
+//===
+//===   You should have received a copy of the GNU General Public License
+//===   along with this program; if not, write to the Free Software
+//===   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+//===
+//===   Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
+//===   Rome - Italy. email: geonetwork@osgeo.org
+//==============================================================================
+package org.fao.geonet.services.statistics;
+
+import jeeves.resources.dbms.Dbms;
+import jeeves.server.ServiceConfig;
+import jeeves.server.context.ServiceContext;
+
+import org.fao.geonet.constants.Geonet;
+import org.fao.geonet.services.NotInReadOnlyModeService;
+import org.jdom.Element;
+
+/**
+ * Service to get main statistics about the content of the catalog
+ * <ul>
+ * <li>total number of searches</li>
+ * </ul>
+ * 
+ */
+public class ContentStatistics extends NotInReadOnlyModeService {
+
+    private static final String NUMBER_OF_METADATA_QUERY = 
+            "SELECT COUNT(*) AS total " +
+            "FROM metadata WHERE isTemplate = ?";
+
+    private static final String NUMBER_OF_HARVESTED_METADATA_QUERY = 
+            "SELECT COUNT(*) AS total " +
+            "FROM metadata WHERE isTemplate = ? AND isHarvested = 'y'";
+
+    private static final String NUMBER_OF_PUBLIC_METADATA_RECORD = 
+            "SELECT COUNT(*) as total FROM metadata m, operationallowed o " +
+            "WHERE m.id = o.metadataid AND groupid = 0 AND operationid = 0 " +
+            "AND isTemplate = ?";
+
+    public void init(String appPath, ServiceConfig params) throws Exception {
+        super.init(appPath, params);
+    }
+
+    @Override
+    public Element serviceSpecificExec(Element params, ServiceContext context) throws Exception {
+        Dbms dbms = (Dbms) context.getResourceManager().open(Geonet.Res.MAIN_DB);
+        Element response = new Element("response");
+
+        // Add parameter by source catalog
+        // Add parameter by group and owner
+
+
+        // Total number of metadata by type
+        SearchStatistics.addSingleDBValueToElement(dbms, response, NUMBER_OF_METADATA_QUERY, "nb_metadata", "total", "n");
+        SearchStatistics.addSingleDBValueToElement(dbms, response, NUMBER_OF_HARVESTED_METADATA_QUERY, "nb_harvested", "total", "n");
+        SearchStatistics.addSingleDBValueToElement(dbms, response, NUMBER_OF_METADATA_QUERY, "nb_template", "total", "y");
+        SearchStatistics.addSingleDBValueToElement(dbms, response, NUMBER_OF_METADATA_QUERY, "nb_subtemplate", "total", "s");
+        SearchStatistics.addSingleDBValueToElement(dbms, response, NUMBER_OF_PUBLIC_METADATA_RECORD, "nb_metadata_public", "total", "n");
+
+        return response;
+    }
+}

services/src/main/java/org/fao/geonet/services/statistics/MetadataStatistics.java

@@ -0,0 +1,120 @@
+//=============================================================================
+//===   Copyright (C) 2001-2013 Food and Agriculture Organization of the
+//===   United Nations (FAO-UN), United Nations World Food Programme (WFP)
+//===   and United Nations Environment Programme (UNEP)
+//===
+//===   This program is free software; you can redistribute it and/or modify
+//===   it under the terms of the GNU General Public License as published by
+//===   the Free Software Foundation; either version 2 of the License, or (at
+//===   your option) any later version.
+//===
+//===   This program is distributed in the hope that it will be useful, but
+//===   WITHOUT ANY WARRANTY; without even the implied warranty of
+//===   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//===   General Public License for more details.
+//===
+//===   You should have received a copy of the GNU General Public License
+//===   along with this program; if not, write to the Free Software
+//===   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+//===
+//===   Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
+//===   Rome - Italy. email: geonetwork@osgeo.org
+//==============================================================================
+package org.fao.geonet.services.statistics;
+
+import jeeves.resources.dbms.Dbms;
+import jeeves.server.ServiceConfig;
+import jeeves.server.context.ServiceContext;
+import jeeves.utils.Util;
+
+import org.fao.geonet.constants.Geonet;
+import org.fao.geonet.services.NotInReadOnlyModeService;
+import org.jdom.Element;
+
+/**
+ * Service to get statistics on metadata record
+ */
+public class MetadataStatistics extends NotInReadOnlyModeService {
+
+    private static final String METADATA_BY_CATEGORY_QUERY = "SELECT categoryid as id, label, count(*) AS total " 
+            + "FROM metadata m, metadatacateg mc, categoriesdes c " 
+            + "WHERE m.id = mc.metadataid AND mc.categoryid = c.iddes AND langid = ? AND isTemplate = ? "
+            + "GROUP BY categoryid, label";
+    private static final String METADATA_BY_GROUP_QUERY = "SELECT groupowner as id, label, count(*) AS total " 
+            + "FROM metadata m, groupsdes g " 
+            + "WHERE m.groupowner = g.iddes AND langid = ? AND isTemplate = ? "
+            + "GROUP BY groupowner, label";
+    private static final String METADATA_BY_OWNER_QUERY = "SELECT owner as id, CONCAT(name, ' ', surname) as label, count(*) AS total "
+            + "FROM metadata m, users u " 
+            + "WHERE m.owner = u.id AND isTemplate like ? "
+            + "GROUP BY owner, username";
+    private static final String METADATA_BY_SOURCE_QUERY = "SELECT source as id, name as label, count(*) AS total " 
+            + "FROM metadata m, sources s " 
+            + "WHERE m.source = s.uuid AND isTemplate like ? "
+            + "GROUP BY source, name";
+    private static final String METADATA_BY_SCHEMA_QUERY = "SELECT schemaid as label, count(*) AS total FROM metadata " 
+            + "WHERE isTemplate like ? "
+            + "GROUP BY schemaid";
+    private static final String METADATA_BY_TEMPLATE_QUERY = "SELECT istemplate as label, count(*) AS total FROM metadata " 
+            + "WHERE isTemplate like ? "
+            + "GROUP BY istemplate";
+    private static final String METADATA_BY_HARVEST_QUERY = "SELECT isharvested as label, count(*) AS total FROM metadata " 
+            + "WHERE isTemplate like ? "
+            + "GROUP BY isharvested";
+
+    // TODO: Could be relevant to add metadata with no status ?
+    private static final String METADATA_BY_STATUS_QUERY = 
+            "SELECT statusid, label, COUNT(*) as total FROM ( " +
+                                    "SELECT statusid, label, m.id, max(s.changedate) " +
+                                    "FROM metadata m, metadatastatus s, statusvaluesdes d " +
+                                    "WHERE m.id = s.metadataid AND s.statusid = d.iddes " +
+                                    "AND d.langid = ? AND isTemplate = ? " +
+                                    "GROUP BY statusid, label, m.id " +
+            ") AS Q GROUP BY statusid, label " + 
+            "UNION " +
+            "SELECT -1, 'nostatus' AS LABEL, COUNT(*) FROM metadata WHERE isTemplate = ? AND id NOT IN (SELECT metadataid FROM metadatastatus)";
+
+    private static final String METADATA_BY_VALIDATION_QUERY = 
+            "SELECT 'invalid' AS label, COUNT(*) AS total FROM (SELECT m.id FROM metadata m, validation v " +
+                                    "WHERE m.id = v.metadataid AND isTemplate = ? " +
+                                    "GROUP BY m.id " +
+                                    "HAVING SUM(status) = 0) AS Q " +
+            "UNION  " +
+            "SELECT 'valid' AS label, COUNT(*) AS total FROM (SELECT m.id FROM metadata m, validation v " +
+                                    "WHERE m.id = v.metadataid AND isTemplate = ? " +
+                                    "GROUP BY m.id " +
+                                    "HAVING SUM(status) > 0) AS Q " +
+            "UNION " +
+            "SELECT 'unchecked' AS label, COUNT(*) AS total FROM metadata m WHERE isTemplate = ? AND id NOT IN " +
+                                    "(SELECT metadataid FROM validation)";
+
+    public void init(String appPath, ServiceConfig params) throws Exception {
+        super.init(appPath, params);
+    }
+
+    @Override
+    public Element serviceSpecificExec(Element params, ServiceContext context) throws Exception {
+        String isTemplate = Util.getParam(params, "isTemplate", "n");
+        String type = Util.getParam(params, "by", "group");
+        Dbms dbms = (Dbms) context.getResourceManager().open(Geonet.Res.MAIN_DB);
+        if (type.equals("owner")) {
+            return dbms.select(METADATA_BY_OWNER_QUERY, isTemplate);
+        } else if (type.equals("source")) {
+            return dbms.select(METADATA_BY_SOURCE_QUERY, isTemplate);
+        } else if (type.equals("schema")) {
+            return dbms.select(METADATA_BY_SCHEMA_QUERY, isTemplate);
+        } else if (type.equals("template")) {
+            return dbms.select(METADATA_BY_TEMPLATE_QUERY, isTemplate);
+        } else if (type.equals("harvested")) {
+            return dbms.select(METADATA_BY_HARVEST_QUERY, isTemplate);
+        } else if (type.equals("category")) {
+            return dbms.select(METADATA_BY_CATEGORY_QUERY, context.getLanguage(), isTemplate);
+        } else if (type.equals("status")) {
+            return dbms.select(METADATA_BY_STATUS_QUERY, context.getLanguage(), isTemplate, isTemplate);
+        } else if (type.equals("validity")) {
+            return dbms.select(METADATA_BY_VALIDATION_QUERY, isTemplate, isTemplate, isTemplate);
+        } else {
+            return dbms.select(METADATA_BY_GROUP_QUERY, context.getLanguage(), isTemplate);
+        }
+    }
+}

services/src/main/java/org/fao/geonet/services/statistics/RequestsByDate.java

@@ -309,7 +309,6 @@ public static void main(String[] args) throws Exception {
 
         JFreeChart chart = ChartFactory.getTimeSeriesChart(dataset, "toto", "titi", "MM/yy", true, true);
         ChartFactory.writeChartImage(chart, new File("/tmp/toto.png"), 600, 400, true, "imageMapName");
-        System.out.println("done.");
     }
 
     /**

services/src/main/java/org/fao/geonet/services/statistics/RequestsByDateStatistics.java

@@ -0,0 +1,148 @@
+//=============================================================================
+//===   Copyright (C) 2001-2013 Food and Agriculture Organization of the
+//===   United Nations (FAO-UN), United Nations World Food Programme (WFP)
+//===   and United Nations Environment Programme (UNEP)
+//===
+//===   This program is free software; you can redistribute it and/or modify
+//===   it under the terms of the GNU General Public License as published by
+//===   the Free Software Foundation; either version 2 of the License, or (at
+//===   your option) any later version.
+//===
+//===   This program is distributed in the hope that it will be useful, but
+//===   WITHOUT ANY WARRANTY; without even the implied warranty of
+//===   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//===   General Public License for more details.
+//===
+//===   You should have received a copy of the GNU General Public License
+//===   along with this program; if not, write to the Free Software
+//===   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+//===
+//===   Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
+//===   Rome - Italy. email: geonetwork@osgeo.org
+//==============================================================================
+package org.fao.geonet.services.statistics;
+
+import java.util.Hashtable;
+import java.util.List;
+
+import jeeves.constants.Jeeves;
+import jeeves.resources.dbms.Dbms;
+import jeeves.server.ServiceConfig;
+import jeeves.server.context.ServiceContext;
+import jeeves.utils.Log;
+import jeeves.utils.Util;
+
+import org.fao.geonet.constants.Geonet;
+import org.fao.geonet.services.NotInReadOnlyModeService;
+import org.jdom.Element;
+
+/**
+ * Service to get the db-stored requests information group by a date (year, month, day)
+ * 
+ * @author nicolas Ribot
+ * 
+ */
+public class RequestsByDateStatistics extends NotInReadOnlyModeService {
+    public static final String BY_YEAR = "YEAR";
+    public static final String BY_MONTH = "MONTH";
+    public static final String BY_DAY = "DAY";
+    public static final String BY_HOUR = "HOUR";
+
+    /** the date to search for from (format MUST be: ) */
+    private String dateFrom;
+    /** the date to search for too (format MUST be: yyy-MM-ddThh:mm) */
+    private String dateTo;
+    /** the type of graphic (by year, month or day to display */
+    private String graphicType;
+    /** to return service statistics by service type (ie. csw, oai, q, ...)*/
+    private boolean byType = false;
+
+    /** the custom part of the date query; according to user choice for graphic */
+    public Hashtable<String, String> queryFragments;
+
+
+    public void init(String appPath, ServiceConfig params) throws Exception {
+        super.init(appPath, params);
+
+        queryFragments = new Hashtable<String, String>(4);
+
+        queryFragments.put(RequestsByDateStatistics.BY_HOUR, "substring(requestDate, 1, 13)");
+        queryFragments.put(RequestsByDateStatistics.BY_DAY, "substring(requestDate, 1, 10)");
+        queryFragments.put(RequestsByDateStatistics.BY_MONTH, "substring(requestDate, 1, 7)");
+        queryFragments.put(RequestsByDateStatistics.BY_YEAR, "substring(requestDate, 1, 4)");
+    }
+
+    // --------------------------------------------------------------------------
+    // ---
+    // --- Service
+    // ---
+    // --------------------------------------------------------------------------
+    @Override
+    public Element serviceSpecificExec(Element params, ServiceContext context) throws Exception {
+        this.dateFrom = Util.getParam(params, "dateFrom");
+        this.dateTo = Util.getParam(params, "dateTo");
+        this.graphicType = Util.getParam(params, "graphicType");
+        this.byType = Util.getParam(params, "byType", false);
+
+        Dbms dbms = (Dbms) context.getResourceManager().open(Geonet.Res.MAIN_DB);
+        Element elResp = new Element(Jeeves.Elem.RESPONSE);
+
+
+        // TODO : if ByServiceType
+        if (byType) {
+            String serviceTypesQuery = "SELECT DISTINCT(service) as type FROM Requests";
+            Element serviceTypes = dbms.select(serviceTypesQuery);
+            for (Object o : serviceTypes.getChildren()) {
+                Element type = (Element)o;
+                String serviceType = type.getChildText("type");
+                String query = buildQuery(serviceType);
+                if (Log.isDebugEnabled(Geonet.SEARCH_LOGGER)) {
+                    Log.debug(Geonet.SEARCH_LOGGER, "query to get count by date:\n" + query);
+                }
+                Element results = dbms.select(query, this.dateFrom, this.dateTo, serviceType);
+                results.setAttribute("service", serviceType);
+                elResp.addContent(results);
+            }
+        } else {
+            String query = buildQuery(null);
+            if (Log.isDebugEnabled(Geonet.SEARCH_LOGGER)) {
+                Log.debug(Geonet.SEARCH_LOGGER, "query to get count by date:\n" + query);
+            }
+            Element results = dbms.select(query, this.dateFrom, this.dateTo);
+            elResp.addContent(results);
+        }
+
+
+        SearchStatistics.addSingleDBValueToElement(dbms, elResp, 
+                "SELECT min(requestdate) AS min FROM Requests", 
+                "dateMin", "min", null);
+        SearchStatistics.addSingleDBValueToElement(dbms, elResp, 
+                "SELECT max(requestdate) AS max FROM Requests", 
+                "dateMax", "max", null);
+
+        elResp.addContent(new Element("dateFrom").setText(this.dateFrom));
+        elResp.addContent(new Element("dateTo").setText(this.dateTo));
+
+
+        return elResp;
+    }
+
+    public String buildQuery(String service) {
+        String requestDateSubstring = this.queryFragments.get(this.graphicType);
+
+        StringBuilder query = new StringBuilder("SELECT ");
+        query.append(requestDateSubstring);
+        query.append(" as reqdate, count(*) as number FROM Requests ");
+        query.append(" where requestdate >= ?");
+        query.append(" and requestdate <= ?");
+        if (service != null) {
+            query.append(" and service = ?");
+        }
+        query.append(" GROUP BY ");
+        query.append(requestDateSubstring);
+        query.append(" ORDER BY ");
+        query.append(requestDateSubstring);
+
+        return query.toString();
+    }
+}