3480 worldmap links

doc/sphinx-guides/source/admin/geoconnect-worldmap.rst

@@ -0,0 +1,25 @@
+Geoconnect and WorldMap
+=======================
+
+.. contents:: :local:
+
+One of the optional components listed under "Architecture and Components" in the :doc:`/installation/prep` section of the Installation Guide is `Geoconnect <https://github.com/IQSS/geoconnect>`_, piece of middleware that allows Dataverse users to create maps in `WorldMap <http://worldmap.harvard.edu>`_ based on geospatial data stored in Dataverse. For more details on the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide.
+
+Removing Dead Explore Links
+---------------------------
+
+After a map has been created in WorldMap (assuming all the setup has been done), an "Explore" button will appear next to the name of the file in Dataverse. The "Explore" button should open the map in WorldMap. In rare occasions, the map has been deleted on the WorldMap side such that the "Explore" button goes nowhere, resulting in a dead link, a 404.
+
+Functionality has been added on the Dataverse side to iterate through all the maps Dataverse knows about (stored in the ``maplayermetadata`` database table) and to check for the existence of each map in WorldMap. The status code returned from WorldMap (200, 404, etc.) is recorded in Dataverse along with a timestamp of when the check was performed. To perform this check, you can execute the following ``curl`` command:
+
+``curl -X POST http://localhost:8080/api/admin/geoconnect/mapLayerMetadatas/check``
+
+The output above will contain the ``layerLink`` being checked as well as the HTTP response status code (200, 404, etc.) in the ``lastVerifiedStatus`` field. 200 means OK and 404 means not found. 500 might indicate that the map is only temporarily unavailable. The ``lastVerifiedStatus`` and ``lastVerifiedTime`` will be persisted to the ``maplayermetadata`` database table.
+
+Armed with this information about WorldMap returning a 404 for a map, you may want to delete any record of the map on the Dataverse side so that the "Explore" button goes away (and so that thumbnail files are cleaned up). To accomplish this, use the following ``curl`` command, substituting the id of the file:
+
+``curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://localhost:8080/api/files/{file_id}/map``
+
+End users can also run the ``DELETE`` command above (if they have permission to edit the dataset) but it's more likely that the sysadmin reading this guide will run it for them. It is recommended that you add the "check" command above to cron so that Dataverse periodically checks on the status of maps in WorldMap. In addition to the "check all maps" command above, you can also check an individual map with the following ``curl`` command, substituting the id of the row from the ``maplayermetadata`` table:
+
+``curl -X POST http://localhost:8080/api/admin/geoconnect/mapLayerMetadatas/check/{id}``

doc/sphinx-guides/source/admin/index.rst

@@ -18,4 +18,5 @@ Contents:
    harvestserver
    metadataexport
    timers
+   geoconnect-worldmap
    troubleshooting

scripts/database/upgrades/upgrade_v4.6.1_to_v4.6.2.sql

@@ -1 +1,3 @@
 ALTER TABLE dataset ADD COLUMN useGenericThumbnail boolean;
+ALTER TABLE maplayermetadata ADD COLUMN lastverifiedtime timestamp without time zone;
+ALTER TABLE maplayermetadata ADD COLUMN lastverifiedstatus bigint;

src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java

@@ -143,6 +143,9 @@ public class EjbDataverseEngine {
     @EJB
     DatasetVersionServiceBean datasetVersionService;
 
+    @EJB
+    MapLayerMetadataServiceBean mapLayerMetadata;
+
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     private EntityManager em;
 
@@ -412,6 +415,11 @@ public DatasetVersionServiceBean datasetVersion() {
                     return datasetVersionService;
                 }
 
+                @Override
+                public MapLayerMetadataServiceBean mapLayerMetadata() {
+                    return mapLayerMetadata;
+                }
+
             };
         }
 

src/main/java/edu/harvard/iq/dataverse/MapLayerMetadata.java

@@ -7,6 +7,7 @@
 package edu.harvard.iq.dataverse;
 
 import java.io.Serializable;
+import java.sql.Timestamp;
 import java.util.Arrays;
 import java.util.List;
 import javax.persistence.Column;
@@ -17,17 +18,20 @@
 import javax.persistence.Index;
 import javax.persistence.JoinColumn;
 import javax.persistence.ManyToOne;
+import javax.persistence.NamedQueries;
+import javax.persistence.NamedQuery;
 import javax.persistence.Table;
 import javax.persistence.Transient;
-import javax.persistence.UniqueConstraint;
-import javax.persistence.Version;
 import org.hibernate.validator.constraints.NotBlank;
 
 /**
  *  File metadata: specifically WorldMap layer information for a specific DataFile 
  * 
  * @author raprasad
  */
+@NamedQueries({
+    @NamedQuery(name = "MapLayerMetadata.findAll",
+            query = "SELECT mlm FROM MapLayerMetadata mlm"),})
 @Entity
 @Table(indexes = {@Index(columnList="dataset_id")})
 public class MapLayerMetadata implements Serializable {
@@ -99,8 +103,19 @@ public class MapLayerMetadata implements Serializable {
     @Column(columnDefinition = "TEXT")
     private String mapLayerLinks;
 
-
-
+    /**
+     * The HTTP Status code (200, 404, etc.) returned when you check to see if
+     * the map/layer exists on the WorldMap side.
+     */
+    @Column(nullable=true)
+    private int lastVerifiedStatus;
+
+    /**
+     * The time that lastVerifiedStatus was last recorded.
+     */
+    @Column(nullable=true)
+    private Timestamp lastVerifiedTime;
+
     /**
      * Get property layerName.
      * @return value of property layerName.
@@ -263,7 +278,21 @@ public void setId(Long id) {
         this.id = id;
     }
 
+    public Timestamp getLastVerifiedTime() {
+        return lastVerifiedTime;
+    }
+
+    public void setLastVerifiedTime(Timestamp lastVerifiedTime) {
+        this.lastVerifiedTime = lastVerifiedTime;
+    }
+
+    public int getLastVerifiedStatus() {
+        return lastVerifiedStatus;
+    }
 
+    public void setLastVerifiedStatus(int lastVerifiedStatus) {
+        this.lastVerifiedStatus = lastVerifiedStatus;
+    }
 
     @Override
     public String toString() {
@@ -272,5 +301,4 @@ public String toString() {
         //return "WorldMap Layer: " + this.layerName + " for DataFile: " + this.dataFile.toString();
     }
 
-
 }

src/main/java/edu/harvard/iq/dataverse/MapLayerMetadataServiceBean.java

@@ -308,5 +308,12 @@ public boolean retrieveMapImageForIcon(MapLayerMetadata mapLayerMetadata) throws
         logger.info("Done");
         return true;
     }   
+
+    public List<MapLayerMetadata> findAll() {
+        TypedQuery<MapLayerMetadata> typedQuery = em.createNamedQuery("MapLayerMetadata.findAll", MapLayerMetadata.class);
+        List<MapLayerMetadata> mapLayerMetadatas = typedQuery.getResultList();
+        return mapLayerMetadatas;
+    }
+
 }
 

src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java

@@ -10,6 +10,7 @@
 import edu.harvard.iq.dataverse.DataverseServiceBean;
 import edu.harvard.iq.dataverse.DvObject;
 import edu.harvard.iq.dataverse.EjbDataverseEngine;
+import edu.harvard.iq.dataverse.MapLayerMetadataServiceBean;
 import edu.harvard.iq.dataverse.MetadataBlock;
 import edu.harvard.iq.dataverse.MetadataBlockServiceBean;
 import edu.harvard.iq.dataverse.PermissionServiceBean;
@@ -33,6 +34,7 @@
 import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean;
 import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.json.JsonParser;
 import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
 import edu.harvard.iq.dataverse.validation.BeanValidationServiceBean;
@@ -190,9 +192,15 @@ String getWrappedMessageWhenJson() {
     @EJB
     protected DatasetVersionServiceBean datasetVersionSvc;
 
-	@PersistenceContext(unitName = "VDCNet-ejbPU")
-	protected EntityManager em;
-
+    @EJB
+    protected MapLayerMetadataServiceBean mapLayerMetadataSrv;
+
+    @EJB
+    protected SystemConfig systemConfig;
+
+    @PersistenceContext(unitName = "VDCNet-ejbPU")
+    protected EntityManager em;
+
     @Context
     protected HttpServletRequest httpRequest;
 

src/main/java/edu/harvard/iq/dataverse/api/Files.java

@@ -1,41 +1,46 @@
-/*
- * To change this license header, choose License Headers in Project Properties.
- * To change this template file, choose Tools | Templates
- * and open the template in the editor.
- */
 package edu.harvard.iq.dataverse.api;
 
-//import com.sun.jersey.core.header.FormDataContentDisposition;
-//import com.sun.jersey.multipart.FormDataParam;
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
+import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DataFileServiceBean;
 import edu.harvard.iq.dataverse.DatasetServiceBean;
 import edu.harvard.iq.dataverse.DatasetVersionServiceBean;
 import edu.harvard.iq.dataverse.DataverseRequestServiceBean;
 import edu.harvard.iq.dataverse.DataverseServiceBean;
 import edu.harvard.iq.dataverse.EjbDataverseEngine;
+import edu.harvard.iq.dataverse.MapLayerMetadata;
 import edu.harvard.iq.dataverse.UserNotificationServiceBean;
+import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.datasetutility.AddReplaceFileHelper;
 import edu.harvard.iq.dataverse.datasetutility.DataFileTagException;
 import edu.harvard.iq.dataverse.datasetutility.NoFilesException;
 import edu.harvard.iq.dataverse.datasetutility.OptionalFileParams;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.impl.DeleteMapLayerMetadataCommand;
 import edu.harvard.iq.dataverse.ingest.IngestServiceBean;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
 import java.io.InputStream;
+import java.sql.Timestamp;
 import java.util.ResourceBundle;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import javax.ejb.EJB;
 import javax.inject.Inject;
 import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
+import static javax.ws.rs.core.Response.Status.FORBIDDEN;
+import static javax.ws.rs.core.Response.Status.NOT_FOUND;
 import org.glassfish.jersey.media.multipart.FormDataBodyPart;
 import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
 import org.glassfish.jersey.media.multipart.FormDataParam;
@@ -216,8 +221,26 @@ public Response replaceFileInDataset(
 
     } // end: replaceFileInDataset
 
-
+    @DELETE
+    @Path("{id}/map")
+    public Response getMapLayerMetadatas(@PathParam("id") Long idSupplied) {
+        DataverseRequest dataverseRequest = null;
+        try {
+            dataverseRequest = createDataverseRequest(findUserOrDie());
+        } catch (WrappedResponse wr) {
+            return error(BAD_REQUEST, "Couldn't find user to execute command: " + wr.getLocalizedMessage());
+        }
+        DataFile dataFile = fileService.find(idSupplied);
+        try {
+            boolean deleted = engineSvc.submit(new DeleteMapLayerMetadataCommand(dataverseRequest, dataFile));
+            if (deleted) {
+                return ok("Map deleted from file id " + dataFile.getId());
+            } else {
+                return error(BAD_REQUEST, "Could not delete map from file id " + dataFile.getId());
+            }
+        } catch (CommandException ex) {
+            return error(BAD_REQUEST, "Problem trying to delete map from file id " + dataFile.getId() + ": " + ex.getLocalizedMessage());
+        }
+    }
 
 }
-
-

src/main/java/edu/harvard/iq/dataverse/api/Geoconnect.java

@@ -0,0 +1,78 @@
+package edu.harvard.iq.dataverse.api;
+
+import com.mashape.unirest.http.Unirest;
+import com.mashape.unirest.http.exceptions.UnirestException;
+import com.mashape.unirest.request.GetRequest;
+import edu.harvard.iq.dataverse.MapLayerMetadata;
+import java.sql.Timestamp;
+import java.util.Date;
+import java.util.logging.Logger;
+import javax.json.Json;
+import javax.json.JsonArrayBuilder;
+import javax.json.JsonObjectBuilder;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.core.Response;
+import static javax.ws.rs.core.Response.Status.NOT_FOUND;
+
+/**
+ * @todo consolidate with WorldMapRelatedData (/api/worldmap)?
+ */
+@Path("admin/geoconnect")
+public class Geoconnect extends AbstractApiBean {
+
+    private static final Logger logger = Logger.getLogger(Geoconnect.class.getCanonicalName());
+
+    @POST
+    @Path("mapLayerMetadatas/check")
+    public Response checkMapLayerMetadatas() {
+        JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();
+        mapLayerMetadataSrv.findAll().stream().map((unmodified) -> checkStatus(unmodified)).forEach((jsonObjectBuilder) -> {
+            jsonArrayBuilder.add(jsonObjectBuilder);
+        });
+        return ok(jsonArrayBuilder);
+    }
+
+    @POST
+    @Path("mapLayerMetadatas/check/{id}")
+    public Response checkMapLayerMetadatas(@PathParam("id") Long idSupplied) {
+        MapLayerMetadata mapLayerMetadata = mapLayerMetadataSrv.find(idSupplied);
+        if (mapLayerMetadata == null) {
+            return error(NOT_FOUND, "Could not find MapLayerMetadata based on id " + idSupplied);
+        }
+        return ok(checkStatus(mapLayerMetadata));
+    }
+
+    private JsonObjectBuilder checkStatus(MapLayerMetadata unmodified) {
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        jsonObjectBuilder.add("fileId", unmodified.getDataFile().getId());
+        jsonObjectBuilder.add("mapLayerMetadataId", unmodified.getId());
+        jsonObjectBuilder.add("layerLink", unmodified.getLayerLink());
+        jsonObjectBuilder.add("fileLandingPage", systemConfig.getDataverseSiteUrl() + "/file.xhtml?fileId=" + unmodified.getDataFile().getId());
+        MapLayerMetadata modified = updateLastVerifiedTimeAndStatusCode(unmodified);
+        if (modified != null) {
+            jsonObjectBuilder.add("lastVerifiedStatus", modified.getLastVerifiedStatus());
+        } else {
+            jsonObjectBuilder.add("error", "Could not check status of map associate with file id " + unmodified.getDataFile().getId());
+        }
+        return jsonObjectBuilder;
+    }
+
+    private MapLayerMetadata updateLastVerifiedTimeAndStatusCode(MapLayerMetadata mapLayerMetadata) {
+        String layerLink = mapLayerMetadata.getLayerLink();
+        GetRequest getRequest = Unirest.get(layerLink);
+        try {
+            int statusCode = getRequest.asBinary().getStatus();
+            mapLayerMetadata.setLastVerifiedStatus(statusCode);
+            Timestamp now = new Timestamp(new Date().getTime());
+            mapLayerMetadata.setLastVerifiedTime(now);
+            logger.fine("Setting status code to " + statusCode + " and timestamp to " + now + " for MapLayerMetadata id " + mapLayerMetadata.getId() + ".");
+            return mapLayerMetadataSrv.save(mapLayerMetadata);
+        } catch (UnirestException ex) {
+            logger.info("Couldn't update last verfied status code or timestamp: " + ex.getLocalizedMessage());
+            return null;
+        }
+    }
+
+}

src/main/java/edu/harvard/iq/dataverse/api/WorldMapRelatedData.java

@@ -56,6 +56,10 @@
  * 
  *  https://github.com/IQSS/shared-dataverse-information
  * 
+ * @todo Audit these methods to ensure they don't pose a security risk. Consider
+ * changing the installer so that like "admin" this "worldmap" endpoint is
+ * blocked out of the box. See
+ * http://guides.dataverse.org/en/4.6.1/installation/config.html#blocking-api-endpoints
  */
 @Path("worldmap")
 public class WorldMapRelatedData extends AbstractApiBean {

src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java

@@ -17,6 +17,7 @@
 import edu.harvard.iq.dataverse.FeaturedDataverseServiceBean;
 import edu.harvard.iq.dataverse.GuestbookResponseServiceBean;
 import edu.harvard.iq.dataverse.GuestbookServiceBean;
+import edu.harvard.iq.dataverse.MapLayerMetadataServiceBean;
 import edu.harvard.iq.dataverse.search.IndexServiceBean;
 import edu.harvard.iq.dataverse.PermissionServiceBean;
 import edu.harvard.iq.dataverse.RoleAssigneeServiceBean;
@@ -114,4 +115,6 @@ public interface CommandContext {
     public PrivateUrlServiceBean privateUrl();
 
     public DatasetVersionServiceBean datasetVersion();
+
+    public MapLayerMetadataServiceBean mapLayerMetadata();
 }