From 1cf0a88c4ff199a1302c2b1dd36403a416b13e00 Mon Sep 17 00:00:00 2001 From: jdeolive Date: Wed, 23 Nov 2016 14:38:39 -0700 Subject: [PATCH] GEOS-7878: Detecting the version of the style being imported. - fix assembly plugin deps - more logging on OWC context - fix importer issues on NetCDF files --- src/community/release/ext-wps-remote.xml | 52 +++-- .../wps/remote/RemoteProcessClient.java | 187 ++++++++++-------- .../wps/remote/plugin/XMPPClient.java | 25 +-- .../remote/plugin/XMPPCompletedMessage.java | 1 - .../plugin/output/XMPPRawDataOutput.java | 48 ++++- .../java/org/geoserver/importer/Importer.java | 1 + .../geoserver/importer/ImporterDataTest.java | 55 +++++- 7 files changed, 213 insertions(+), 156 deletions(-) diff --git a/src/community/release/ext-wps-remote.xml b/src/community/release/ext-wps-remote.xml index a87da4cc439..f3dbf50bd31 100644 --- a/src/community/release/ext-wps-remote.xml +++ b/src/community/release/ext-wps-remote.xml @@ -10,42 +10,34 @@ gs-wps-remote*.jar - bcprov*.jar - common*.jar - dnsjava*.jar - ehcache*.jar - freemarker*.jar - geodb*.jar - GeographicLib*.jar + *commons-beanutils*.jar + *commons-exec*.jar + *commons-vfs2*.jar + *dnsjava*.jar + *geodb*.jar *importer*.jar - *geojson*.jar - hamcrest*.jar - hatbox*.jar - http*.jar - icu4j*.jar - imageio*.jar - jasypt*.jar - javacsv*.jar - jbosh*.jar - jdom*.jar - jetty*.jar - json*.jar - jgridshift*.jar - jmf*.jar - jsqlparser*.jar - jsr*.jar - jt*.jar - mail*.jar - mina*.jar - picocontainer*.jar - *smack*.jar + *hamcrest*.jar + *hatbox*.jar + *httpclient*.jar + *httpcore*.jar + *icu4j*.jar + *javacsv*.jar + *jbosh*.jar + *jetty*.jar + *jmf*.jar + *jsqlparser*.jar + *jtds*.jar + *jzlib*.jar *pyrolite*.jar + *smack*.jar *spark*.jar + *mina*.jar + *nbxml*.jar + *plexus*.jar + *regexp*.jar *vysper*.jar *xep*.jar *xmpp*.jar - xml*.jar - xpp3*.jar diff --git a/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/RemoteProcessClient.java b/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/RemoteProcessClient.java index 25279ed0813..88366584d9c 100644 --- a/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/RemoteProcessClient.java +++ b/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/RemoteProcessClient.java @@ -25,6 +25,7 @@ import org.geoserver.catalog.ResourceInfo; import org.geoserver.catalog.StyleInfo; import org.geoserver.catalog.WorkspaceInfo; +import org.geoserver.catalog.impl.CoverageInfoImpl; import org.geoserver.catalog.impl.DimensionInfoImpl; import org.geoserver.config.GeoServer; import org.geoserver.importer.ImportContext; @@ -308,22 +309,21 @@ public LayerInfo importLayer(File file, String type, DataStoreInfo store, String : importer.createContext(new SpatialFile(file))); if (context.getTasks() != null && context.getTasks().size() > 0) { + WorkspaceInfo ws = null; ImportTask task = context.getTasks().get(0); if (targetWorkspace != null) { - LOGGER.fine( " - [Remote Process Client - importLayer] Looking for Workspace in the catalog:" + targetWorkspace); - WorkspaceInfo ws = importer.getCatalog().getWorkspaceByName(targetWorkspace); + ws = importer.getCatalog().getWorkspaceByName(targetWorkspace); if (ws != null) { LOGGER.fine(" - [Remote Process Client - importLayer] Workspace found:" + ws); context.setTargetWorkspace(ws); } else { - LOGGER.fine( " - [Remote Process Client - importLayer] Workspace *NOT* found - using the Default one:" + importer.getCatalog().getDefaultWorkspace()); @@ -343,108 +343,127 @@ public LayerInfo importLayer(File file, String type, DataStoreInfo store, String } } - if (name != null) + if (name != null) { task.getLayer().setName(name); - if (title != null) + } + if (title != null) { task.getLayer().setTitle(title); - if (description != null) + } + if (description != null) { task.getLayer().setAbstract(description); - - if (metadata != null) + } + if (metadata != null) { task.getLayer().getMetadata().put("owc_properties", metadata); + } + // AF: Importer ISSUE -> The target workspace is not honored + /*if (task.getLayer().getResource() != null) { + if (ws != null) { + task.getLayer().getResource().getStore().setWorkspace(ws); + } - importer.run(context); - - if (context.getState() == ImportContext.State.COMPLETE) { - if (context.getTasks() != null && context.getTasks().size() > 0) { - // ImportTask task = context.getTasks().get(0); - // assertEquals(ImportTask.State.READY, task.getState()); - - // assertEquals("the layer name", task.getLayer().getResource().getName()); - - task = context.getTasks().get(0); - - // WARNING: The Importer Configures Just The First Layer - if (task.getLayer().getResource() instanceof CoverageInfo) { - CoverageInfo ci = ((CoverageInfo) task.getLayer().getResource()); - - GridCoverageReader reader = null; - try { - reader = ci.getGridCoverageReader(null, null); - - String[] cvNames = reader.getGridCoverageNames(); - - if (cvNames != null && cvNames.length > 0) { - final String nativeCoverageName = cvNames[0]; - - ci.setNativeCoverageName(nativeCoverageName); - - // if(type.equals("application/x-netcdf")) Set Dimensions - if (reader instanceof StructuredGridCoverage2DReader) { - StructuredGridCoverage2DReader structuredReader = ((StructuredGridCoverage2DReader) reader); + if (task.getLayer().getResource() instanceof CoverageInfo) { + ((CoverageInfoImpl) task.getLayer().getResource()).setNativeCoverageName(name); + } + }*/ - // Getting dimension descriptors - final List dimensionDescriptors = structuredReader - .getDimensionDescriptors(nativeCoverageName); - DimensionDescriptor timeDimension = null; - DimensionDescriptor elevationDimension = null; - final List customDimensions = new ArrayList(); + importer.run(context); - // Collect dimension Descriptor info - for (DimensionDescriptor dimensionDescriptor : dimensionDescriptors) { - if (dimensionDescriptor.getName() - .equalsIgnoreCase(ResourceInfo.TIME)) { - timeDimension = dimensionDescriptor; - } else if (dimensionDescriptor.getName() - .equalsIgnoreCase(ResourceInfo.ELEVATION)) { - elevationDimension = dimensionDescriptor; - } else { - customDimensions.add(dimensionDescriptor); + for (int importChecks=0; importChecks<10; importChecks++) { + if (context.getState() == ImportContext.State.COMPLETE) { + if (context.getTasks() != null && context.getTasks().size() > 0) { + // ImportTask task = context.getTasks().get(0); + // assertEquals(ImportTask.State.READY, task.getState()); + + // assertEquals("the layer name", task.getLayer().getResource().getName()); + + task = context.getTasks().get(0); + + // WARNING: The Importer Configures Just The First Layer + if (task.getLayer().getResource() instanceof CoverageInfo) { + CoverageInfo ci = ((CoverageInfo) task.getLayer().getResource()); + + GridCoverageReader reader = null; + try { + reader = ci.getGridCoverageReader(null, null); + + String[] cvNames = reader.getGridCoverageNames(); + + if (cvNames != null && cvNames.length > 0) { + final String nativeCoverageName = cvNames[0]; + + ci.setNativeCoverageName(nativeCoverageName); + + // if(type.equals("application/x-netcdf")) Set Dimensions + if (reader instanceof StructuredGridCoverage2DReader) { + StructuredGridCoverage2DReader structuredReader = ((StructuredGridCoverage2DReader) reader); + + // Getting dimension descriptors + final List dimensionDescriptors = structuredReader + .getDimensionDescriptors(nativeCoverageName); + DimensionDescriptor timeDimension = null; + DimensionDescriptor elevationDimension = null; + final List customDimensions = new ArrayList(); + + // Collect dimension Descriptor info + for (DimensionDescriptor dimensionDescriptor : dimensionDescriptors) { + if (dimensionDescriptor.getName() + .equalsIgnoreCase(ResourceInfo.TIME)) { + timeDimension = dimensionDescriptor; + } else if (dimensionDescriptor.getName() + .equalsIgnoreCase(ResourceInfo.ELEVATION)) { + elevationDimension = dimensionDescriptor; + } else { + customDimensions.add(dimensionDescriptor); + } } - } - final boolean defaultTimeNeeded = timeDimension != null; - final boolean defaultElevationNeeded = elevationDimension != null; + final boolean defaultTimeNeeded = timeDimension != null; + final boolean defaultElevationNeeded = elevationDimension != null; - // Create Default Time Dimension If Needed - if (defaultTimeNeeded) { - DimensionInfo di = new DimensionInfoImpl(); - di.setEnabled(true); - di.setPresentation(DimensionPresentation.LIST); - di.setAttribute(timeDimension.getStartAttribute()); - ci.getMetadata().put(ResourceInfo.TIME, di); - } + // Create Default Time Dimension If Needed + if (defaultTimeNeeded) { + DimensionInfo di = new DimensionInfoImpl(); + di.setEnabled(true); + di.setPresentation(DimensionPresentation.LIST); + di.setAttribute(timeDimension.getStartAttribute()); + ci.getMetadata().put(ResourceInfo.TIME, di); + } - // Create Default Elevation Dimension If Needed - if (defaultElevationNeeded) { - DimensionInfo di = new DimensionInfoImpl(); - di.setEnabled(true); - di.setPresentation(DimensionPresentation.LIST); - di.setUnits("EPSG:5030"); - di.setUnitSymbol("m"); - di.setAttribute(elevationDimension.getStartAttribute()); - ci.getMetadata().put(ResourceInfo.ELEVATION, di); + // Create Default Elevation Dimension If Needed + if (defaultElevationNeeded) { + DimensionInfo di = new DimensionInfoImpl(); + di.setEnabled(true); + di.setPresentation(DimensionPresentation.LIST); + di.setUnits("EPSG:5030"); + di.setUnitSymbol("m"); + di.setAttribute(elevationDimension.getStartAttribute()); + ci.getMetadata().put(ResourceInfo.ELEVATION, di); + } } } + } finally { + // WARNING: Disposing The Reader Causes The Catalog To Fail + /*if (reader != null) { + reader.dispose(); + }*/ } - } finally { - // WARNING: Disposing The Reader Causes The Catalog To Fail - /*if (reader != null) { - reader.dispose(); - }*/ } - } - LOGGER.fine( - " - [Remote Process Client - importLayer] The Importer has finished correctly for Spatial File:" - + file.getAbsolutePath()); + LOGGER.fine( + " - [Remote Process Client - importLayer] The Importer has finished correctly for Spatial File:" + + file.getAbsolutePath()); - return task.getLayer(); + return task.getLayer(); + } else { + break; + } + } else { + Thread.sleep(1500); } } } - LOGGER.fine( + LOGGER.warning( " - [Remote Process Client - importLayer] The Importer has finished *BUT* did not returned any layer for Spatial File:" + file.getAbsolutePath()); diff --git a/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/plugin/XMPPClient.java b/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/plugin/XMPPClient.java index 77374c557fd..36f80d468e2 100644 --- a/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/plugin/XMPPClient.java +++ b/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/plugin/XMPPClient.java @@ -6,7 +6,6 @@ import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.lang.reflect.Array; import java.net.InetAddress; @@ -222,28 +221,8 @@ static enum CType { // ---- PRIMITIVE_NAME_TYPE_MAP.put("application/owc", new Object[] { RawData.class, CType.COMPLEX, - //new StringRawData("", "application/vnd.geo+json"), - new RawData() { - - private String data = ""; - - @Override - public String getMimeType() { - return "application/vnd.geo+json"; - } - - @Override - public InputStream getInputStream() throws IOException { - return IOUtils.toInputStream(data); - } - - @Override - public String getFileExtension() { - return "json"; - } - - }, - "application/vnd.geo+json", ".json" }); + new StringRawData("", "application/vnd.geo+json"), "application/vnd.geo+json", + ".json" }); // ---- PRIMITIVE_NAME_TYPE_MAP.put("image/geotiff", diff --git a/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/plugin/XMPPCompletedMessage.java b/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/plugin/XMPPCompletedMessage.java index fc6710be0ae..dc4342e306f 100644 --- a/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/plugin/XMPPCompletedMessage.java +++ b/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/plugin/XMPPCompletedMessage.java @@ -6,7 +6,6 @@ import java.io.IOException; import java.net.URLDecoder; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; diff --git a/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/plugin/output/XMPPRawDataOutput.java b/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/plugin/output/XMPPRawDataOutput.java index 80dc76dbd16..7c910bf38a0 100644 --- a/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/plugin/output/XMPPRawDataOutput.java +++ b/src/community/wps-remote/src/main/java/org/geoserver/wps/remote/plugin/output/XMPPRawDataOutput.java @@ -41,6 +41,7 @@ import org.geoserver.wps.process.StringRawData; import org.geoserver.wps.remote.WmcFeature; import org.geoserver.wps.remote.plugin.XMPPClient; +import org.geotools.feature.NameImpl; import org.geotools.geometry.jts.ReferencedEnvelope; import org.geotools.util.logging.Logging; @@ -119,17 +120,18 @@ public Object produceOutput(Object value, String type, String pID, String baseUR LOGGER.finest("[XMPP Raw Data Output - ProduceOutput] FileRawData:" + fileName); - final File outputFile = getOutputFile(xmppClient, (String) value); + File outputFile = getOutputFile(xmppClient, (String) value); + if (outputFile.renameTo(new File(outputFile.getParentFile(), fileName))) { + outputFile = new File(outputFile.getParentFile(), fileName); + outputFile.setLastModified(System.nanoTime()); + } + value = new ResourceRawData(Files.asResource(outputFile), ((ResourceRawData) sample).getMimeType(), ((ResourceRawData) sample).getFileExtension()); if (publish) { - final File tempFile = new File(FileUtils.getTempDirectory(), fileName); - FileUtils.copyFile(((ResourceRawData) value).getResource().file(), tempFile); - FileUtils.waitFor(tempFile, 5); - try { - xmppClient.importLayer(tempFile, type, null, name + "_" + pID, title, description, + xmppClient.importLayer(outputFile, type, null, name + "_" + pID, title, description, defaultStyle, targetWorkspace, metadata); } catch (Exception e) { LOGGER.log(Level.WARNING, @@ -258,21 +260,44 @@ private Object encodeAsPlainOWCMapContext(Object value, String type, String pID, String[] workspaces = (targetWorkspace != null ? targetWorkspace.split(";") : null); LOGGER.finest("[XMPP Raw Data Output - ProduceOutput] encodeAsPlainOWCMapContext:" - + layerToPublish); + + value); List wmc = new ArrayList(); if (layerToPublish != null && layerToPublish.length > 0) { - final Catalog catalog = xmppClient.getGeoServer().getCatalog(); + final GeoServer geoServer = xmppClient.getGeoServer(); + final Catalog catalog = geoServer.getCatalog(); for (int fi = 0; fi < layerToPublish.length; fi++) { final String layerBaseName = layerToPublish[fi]; final String layerStyle = styles[fi]; final String layerWorkspace = workspaces[fi]; - LayerInfo layerInfo = catalog.getLayerByName(layerBaseName + "_" + pID); + LOGGER.finest("[XMPP Raw Data Output - ProduceOutput] looking for LayerInfo:" + + layerBaseName + "_" + pID); + + LayerInfo layerInfo = + (catalog.getLayerByName(layerBaseName + "_" + pID) != null ? + catalog.getLayerByName(layerBaseName + "_" + pID) : + catalog.getLayerByName(new NameImpl(layerWorkspace, layerBaseName + "_" + pID))); + + if (layerInfo == null) { + LOGGER.warning("[XMPP Raw Data Output - ProduceOutput] cuold not find LayerInfo [" + + layerBaseName + "_" + pID + "]... going to scan the whole Catalog!"); + for (LayerInfo layer : catalog.getLayers()){ + LOGGER.info("[XMPP Raw Data Output - ProduceOutput] looking for LayerInfo:" + layer.getName()); + if (layer.getName().contains(layerBaseName) && layer.getName().contains(pID)) { + LOGGER.info("[XMPP Raw Data Output - ProduceOutput] found candidate LayerInfo:" + layer.getName()); + layerInfo = layer; + break; + } + } + } if (layerInfo != null) { + LOGGER.finest("[XMPP Raw Data Output - ProduceOutput] found LayerInfo:" + + layerBaseName + "_" + pID); + if (layerStyle.trim().length() > 0) { StyleInfo style = catalog.getStyleByName(layerStyle); @@ -296,6 +321,11 @@ private Object encodeAsPlainOWCMapContext(Object value, String type, String pID, } wmc.add(layerInfo); + LOGGER.finest("[XMPP Raw Data Output - ProduceOutput] added LayerInfo:" + + layerInfo); + } else { + LOGGER.finest("[XMPP Raw Data Output - ProduceOutput] could not find any LayerInfo:" + + layerBaseName + "_" + pID); } } } diff --git a/src/extension/importer/core/src/main/java/org/geoserver/importer/Importer.java b/src/extension/importer/core/src/main/java/org/geoserver/importer/Importer.java index a83ef3e2334..b59bb4ce56e 100644 --- a/src/extension/importer/core/src/main/java/org/geoserver/importer/Importer.java +++ b/src/extension/importer/core/src/main/java/org/geoserver/importer/Importer.java @@ -1692,6 +1692,7 @@ StyleInfo createStyleFromFile(File styleFile, ImportTask task) { info.setFilename(styleName + "." +ext); info.setFormat(styleHandler.getFormat()); + info.setFormatVersion(styleHandler.version(styleFile)); info.setWorkspace(task.getStore().getWorkspace()); try (InputStream in = new FileInputStream(styleFile)) { diff --git a/src/extension/importer/core/src/test/java/org/geoserver/importer/ImporterDataTest.java b/src/extension/importer/core/src/test/java/org/geoserver/importer/ImporterDataTest.java index 23ea54ebf21..ca6445608ae 100644 --- a/src/extension/importer/core/src/test/java/org/geoserver/importer/ImporterDataTest.java +++ b/src/extension/importer/core/src/test/java/org/geoserver/importer/ImporterDataTest.java @@ -25,6 +25,7 @@ import org.geoserver.catalog.LayerInfo; import org.geoserver.catalog.NamespaceInfo; import org.geoserver.catalog.ResourceInfo; +import org.geoserver.catalog.SLDHandler; import org.geoserver.catalog.StyleInfo; import org.geoserver.catalog.WorkspaceInfo; import org.geoserver.importer.ImportTask.State; @@ -1044,7 +1045,50 @@ public void testImportArchiveWithStyleFile() throws Exception { " " + ""; - FileUtils.write(new File(dir, "archsites.sld"), sld); + StyleInfo info = writeStyleAndImport(sld, "archsites.sld", dir); + Style style = info.getStyle(); + + StyleAttributeExtractor atts = new StyleAttributeExtractor(); + style.accept(atts); + + assertTrue(atts.getAttributeNameSet().contains("CAT_ID")); + } + + @Test + public void testImportStyleWithCorrectVersion() throws Exception { + File dir = unpack("shape/archsites_epsg_prj.zip"); + + // add an sld file + String sld = + "" + + "" + + " " + + " " + + " " + + " " + + " " + + " " + + " #d7191c" + + " " + + " " + + " #000000" + + " " + + " " + + " " + + " " + + " " + + " " + + ""; + + StyleInfo info = writeStyleAndImport(sld, "archsites.sld", dir); + assertEquals(SLDHandler.VERSION_11, info.getFormatVersion()); + } + + /* + * Helper for tests exercising the style file import function. + */ + StyleInfo writeStyleAndImport(String sld, String filename, File dir) throws IOException { + FileUtils.write(new File(dir, filename), sld); ImportContext imp = importer.createContext(new Directory(dir)); importer.run(imp); @@ -1052,13 +1096,6 @@ public void testImportArchiveWithStyleFile() throws Exception { assertEquals(ImportContext.State.COMPLETE, imp.getState()); checkNoErrors(imp); - StyleInfo info = - getCatalog().getStyle(imp.getTasks().get(0).getLayer().getDefaultStyle().getId()); - Style style = info.getStyle(); - - StyleAttributeExtractor atts = new StyleAttributeExtractor(); - style.accept(atts); - - assertTrue(atts.getAttributeNameSet().contains("CAT_ID")); + return getCatalog().getStyle(imp.getTasks().get(0).getLayer().getDefaultStyle().getId()); } }