Permalink
Browse files

Merge branch 'kml' into rogue

Conflicts:
	importer/src/main/java/org/opengeo/data/importer/Importer.java
  • Loading branch information...
rmarianski committed Oct 25, 2012
2 parents be3eb2f + 4cb81d2 commit ebbc56074ec0c9c97fc6f548f13d4a49d33842b5
@@ -45,6 +45,7 @@
import org.geotools.data.FeatureWriter;
import org.geotools.data.Transaction;
import org.geotools.data.directory.DirectoryDataStore;
+import org.geotools.data.postgis.PostGISDialect;
import org.geotools.data.shapefile.ShapefileDataStore;
import org.geotools.feature.simple.SimpleFeatureTypeBuilder;
import org.geotools.geometry.GeneralEnvelope;
@@ -775,7 +776,8 @@ void doIndirectImport(ImportTask task, ImportFilter filter) throws IOException {
// the result of a transform. there may be another way...
FeatureTypeInfo resource = getCatalog().getResourceByName(
featureType.getQualifiedName(), FeatureTypeInfo.class);
- if (resource.getNativeBoundingBox().isEmpty()) {
+ if (resource.getNativeBoundingBox().isEmpty()
+ || resource.getMetadata().get("recalculate-bounds") != null) {
// force computation
CatalogBuilder cb = new CatalogBuilder(getCatalog());
ReferencedEnvelope nativeBounds = cb.getNativeBounds(resource);
@@ -838,7 +840,7 @@ void loadIntoDataStore(ImportItem item, DataStoreInfo store, VectorFormat format
ImportData data = item.getTask().getData();
FeatureReader reader = format.read(data, item);
SimpleFeatureType featureType = (SimpleFeatureType) reader.getFeatureType();
- String featureTypeName = featureType.getName().getLocalPart();
+ String featureTypeName = item.getLayer().getName();
DataStore dataStore = (DataStore) store.getDataStore(null);
FeatureDataConverter featureDataConverter = FeatureDataConverter.DEFAULT;
@@ -852,8 +854,18 @@ else if (isOracleDataStore(dataStore)) {
featureType = featureDataConverter.convertType(featureType, format, data, item);
UpdateMode updateMode = item.updateMode();
if (updateMode == null) {
+ if (dataStore instanceof JDBCDataStore
+ && ((JDBCDataStore) dataStore).getSQLDialect() instanceof PostGISDialect) {
+ // trim the length of the name
+ // by default, postgis table/index names need to fit in 64 characters
+ // with the "spatial_" prefix and "_geometry" suffix, that leaves 47 chars
+ // and we should leave room to append integers to make the name unique too
+ if (featureTypeName.length() > 45) {
+ featureTypeName = featureTypeName.substring(0, 45);
+ }
+ }
//find a unique type name in the target store
- featureTypeName = findUniqueNativeFeatureTypeName(featureType, store);
+ featureTypeName = findUniqueNativeFeatureTypeName(featureTypeName, store);
item.setOriginalName(featureType.getTypeName());
if (!featureTypeName.equals(featureType.getTypeName())) {
@@ -1099,9 +1111,12 @@ String findUniqueResourceName(ResourceInfo resource)
}
String findUniqueNativeFeatureTypeName(FeatureType featureType, DataStoreInfo store) throws IOException {
+ return findUniqueNativeFeatureTypeName(featureType.getName().getLocalPart(), store);
+ }
+
+ private String findUniqueNativeFeatureTypeName(String name, DataStoreInfo store) throws IOException {
DataStore dataStore = (DataStore) store.getDataStore(null);
- String name = featureType.getName().getLocalPart();
-
+
//hack for oracle, all names must be upper case
//TODO: abstract this into FeatureConverter
if (isOracleDataStore(dataStore)) {
@@ -1118,7 +1133,7 @@ String findUniqueNativeFeatureTypeName(FeatureType featureType, DataStoreInfo st
i++;
}
}
-
+
return name;
}
@@ -33,7 +33,7 @@
private static final Logger LOGGER = Logging.getLogger(VFSWorker.class);
private static final List<String> extensions = Arrays.asList(".zip", ".tar", ".tar.gz", ".tgz",
- ".tar.bz2", ".tbz2", ".gz", ".bz2", ".jar");
+ ".tar.bz2", ".tbz2", ".gz", ".bz2", ".jar", ".kmz");
public VFSWorker() {
@@ -130,7 +130,7 @@ private String getaArchiveURLProtocol(final File file) {
return "file://";
}
String name = file.getName().toLowerCase();
- if (name.endsWith(".zip")) {
+ if (name.endsWith(".zip") || name.endsWith(".kmz")) {
return "zip://";
}
if (name.endsWith(".tar")) {
@@ -3,8 +3,15 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.commons.io.FilenameUtils;
import org.geoserver.catalog.AttributeTypeInfo;
@@ -13,6 +20,7 @@
import org.geoserver.catalog.CatalogFactory;
import org.geoserver.catalog.FeatureTypeInfo;
import org.geoserver.catalog.LayerInfo;
+import org.geoserver.catalog.MetadataMap;
import org.geoserver.catalog.ResourceInfo;
import org.geoserver.catalog.StoreInfo;
import org.geoserver.catalog.WorkspaceInfo;
@@ -26,7 +34,6 @@
import org.opengeo.data.importer.VectorFormat;
import org.opengeo.data.importer.job.ProgressMonitor;
import org.opengeo.data.importer.transform.KMLPlacemarkTransform;
-import org.opengis.feature.Feature;
import org.opengis.feature.simple.SimpleFeature;
import org.opengis.feature.simple.SimpleFeatureType;
import org.opengis.feature.type.AttributeDescriptor;
@@ -42,6 +49,8 @@
public static CoordinateReferenceSystem KML_CRS;
+ private static KMLPlacemarkTransform kmlTransform = new KMLPlacemarkTransform();
+
private static ReferencedEnvelope EMPTY_BOUNDS = new ReferencedEnvelope();
static {
@@ -57,18 +66,45 @@
@Override
public FeatureReader read(ImportData data, ImportItem item) throws IOException {
File file = getFileFromData(data);
- return read(file);
+
+ // we need to get the feature type, to use for the particular parse through the file
+ // since we put it on the metadata from the list method, we first check if that's still available
+ SimpleFeatureType ft = (SimpleFeatureType) item.getMetadata().get(FeatureType.class);
+ if (ft == null) {
+ // if the type is not available, we can generate one from the resource
+ // we aren't able to ask for the feature type from the resource directly,
+ // because we don't have a backing store
+ FeatureTypeInfo fti = (FeatureTypeInfo) item.getLayer().getResource();
+ SimpleFeatureTypeBuilder ftb = new SimpleFeatureTypeBuilder();
+ ftb.setName(fti.getName());
+ List<AttributeTypeInfo> attributes = fti.getAttributes();
+ for (AttributeTypeInfo attr : attributes) {
+ ftb.add(attr.getName(), attr.getBinding());
+ }
+ ft = ftb.buildFeatureType();
+ MetadataMap metadata = fti.getMetadata();
+ if (metadata.containsKey("importschemanames")) {
+ Map<Object, Object> userData = ft.getUserData();
+ userData.put("schemanames", metadata.get("importschemanames"));
+ }
+ }
+ return read(ft, file);
}
- private FeatureReader<FeatureType, Feature> read(File file) {
+ public FeatureReader<SimpleFeatureType, SimpleFeature> read(SimpleFeatureType featureType,
+ File file) {
try {
- SimpleFeatureType featureType = parseFeatureType(file);
return new KMLTransformingFeatureReader(featureType, new FileInputStream(file));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
+ public FeatureReader<SimpleFeatureType, SimpleFeature> read(SimpleFeatureType featureType,
+ InputStream inputStream) {
+ return new KMLTransformingFeatureReader(featureType, inputStream);
+ }
+
@Override
public void dispose(@SuppressWarnings("rawtypes") FeatureReader reader, ImportItem item)
throws IOException {
@@ -107,67 +143,146 @@ public StoreInfo createStore(ImportData data, WorkspaceInfo workspace, Catalog c
return null;
}
- public SimpleFeatureType parseFeatureType(File file) throws IOException {
- String baseName = FilenameUtils.getBaseName(file.getName());
- FileInputStream fileInputStream = null;
- FeatureReader<FeatureType, Feature> featureReader = null;
+ public Collection<SimpleFeatureType> parseFeatureTypes(String typeName, File file)
+ throws IOException {
+ InputStream inputStream = null;
try {
- fileInputStream = new FileInputStream(file);
- featureReader = new KMLRawFeatureReader(fileInputStream);
- if (!featureReader.hasNext()) {
- throw new IllegalArgumentException("KML file " + file.getName()
- + " has no features");
- }
- SimpleFeature feature = (SimpleFeature) featureReader.next();
- SimpleFeatureType type = feature.getType();
- KMLPlacemarkTransform transform = new KMLPlacemarkTransform();
- SimpleFeatureType transformedType = transform.convertFeatureType(type);
- SimpleFeatureTypeBuilder tb = new SimpleFeatureTypeBuilder();
- tb.init(transformedType);
- tb.setName(baseName);
- tb.setCRS(KML_CRS);
- tb.setSRS(KML_SRS);
- SimpleFeatureType featureType = tb.buildFeatureType();
- return featureType;
+ inputStream = new FileInputStream(file);
+ return parseFeatureTypes(typeName, inputStream);
} finally {
- if (featureReader != null) {
- featureReader.close();
+ if (inputStream != null) {
+ inputStream.close();
}
- if (fileInputStream != null) {
- // closing the feature reader should close the stream as well
- // but in case something went wrong
- fileInputStream.close();
+ }
+ }
+
+ private SimpleFeatureType unionFeatureTypes(SimpleFeatureType a, SimpleFeatureType b) {
+ if (a == null) {
+ return b;
+ }
+ SimpleFeatureTypeBuilder ftb = new SimpleFeatureTypeBuilder();
+ ftb.init(a);
+ List<AttributeDescriptor> attributeDescriptors = a.getAttributeDescriptors();
+ Set<String> attrNames = new HashSet<String>(attributeDescriptors.size());
+ for (AttributeDescriptor ad : attributeDescriptors) {
+ attrNames.add(ad.getLocalName());
+ }
+ for (AttributeDescriptor ad : b.getAttributeDescriptors()) {
+ if (!attrNames.contains(ad.getLocalName())) {
+ ftb.add(ad);
}
}
+ return ftb.buildFeatureType();
+ }
+
+ public SimpleFeatureType convertParsedFeatureType(SimpleFeatureType ft, String name,
+ Set<String> untypedAttributes) {
+ SimpleFeatureType transformedType = kmlTransform.convertFeatureType(ft);
+ SimpleFeatureTypeBuilder ftb = new SimpleFeatureTypeBuilder();
+ ftb.init(transformedType);
+ Set<String> existringAttrNames = new HashSet<String>();
+ for (AttributeDescriptor ad : ft.getAttributeDescriptors()) {
+ existringAttrNames.add(ad.getLocalName());
+ }
+ for (String attr : untypedAttributes) {
+ if (!existringAttrNames.contains(attr)) {
+ ftb.add(attr, String.class);
+ }
+ }
+ ftb.setName(name);
+ ftb.setCRS(KML_CRS);
+ ftb.setSRS(KML_SRS);
+ return ftb.buildFeatureType();
+ }
+
+ public List<SimpleFeatureType> parseFeatureTypes(String typeName, InputStream inputStream)
+ throws IOException {
+ KMLRawReader reader = new KMLRawReader(inputStream,
+ KMLRawReader.ReadType.SCHEMA_AND_FEATURES);
+ Set<String> untypedAttributes = new HashSet<String>();
+ List<String> schemaNames = new ArrayList<String>();
+ List<SimpleFeatureType> schemas = new ArrayList<SimpleFeatureType>();
+ SimpleFeatureType aggregateFeatureType = null;
+ for (Object object : reader) {
+ if (object instanceof SimpleFeature) {
+ SimpleFeature feature = (SimpleFeature) object;
+ SimpleFeatureType ft = feature.getFeatureType();
+ aggregateFeatureType = unionFeatureTypes(aggregateFeatureType, ft);
+ Map<Object, Object> userData = feature.getUserData();
+ @SuppressWarnings("unchecked")
+ Map<String, Object> untypedData = (Map<String, Object>) userData
+ .get("UntypedExtendedData");
+ if (untypedData != null) {
+ untypedAttributes.addAll(untypedData.keySet());
+ }
+ } else if (object instanceof SimpleFeatureType) {
+ SimpleFeatureType schema = (SimpleFeatureType) object;
+ schemas.add(schema);
+ schemaNames.add(schema.getName().getLocalPart());
+ }
+ }
+ if (aggregateFeatureType == null && schemas.isEmpty()) {
+ throw new IllegalArgumentException("No features found");
+ }
+ SimpleFeatureType featureType = aggregateFeatureType;
+ for (SimpleFeatureType schema : schemas) {
+ featureType = unionFeatureTypes(featureType, schema);
+ }
+ featureType = convertParsedFeatureType(featureType, typeName, untypedAttributes);
+ if (!schemaNames.isEmpty()) {
+ Map<Object, Object> userData = featureType.getUserData();
+ userData.put("schemanames", schemaNames);
+ }
+ return Collections.singletonList(featureType);
}
@Override
public List<ImportItem> list(ImportData data, Catalog catalog, ProgressMonitor monitor)
throws IOException {
File file = getFileFromData(data);
CatalogBuilder cb = new CatalogBuilder(catalog);
- SimpleFeatureType featureType = parseFeatureType(file);
+ String baseName = typeNameFromFile(file);
CatalogFactory factory = catalog.getFactory();
- FeatureTypeInfo ftinfo = factory.createFeatureType();
- ftinfo.setEnabled(true);
- String name = featureType.getName().getLocalPart();
- ftinfo.setNativeName(name);
- ftinfo.setName(name);
- ftinfo.setNamespace(catalog.getDefaultNamespace());
- List<AttributeTypeInfo> attributes = ftinfo.getAttributes();
- for (AttributeDescriptor ad : featureType.getAttributeDescriptors()) {
- AttributeTypeInfo att = factory.createAttribute();
- att.setName(ad.getLocalName());
- att.setBinding(ad.getType().getBinding());
- attributes.add(att);
+
+ Collection<SimpleFeatureType> featureTypes = parseFeatureTypes(baseName, file);
+ List<ImportItem> result = new ArrayList<ImportItem>(featureTypes.size());
+ for (SimpleFeatureType featureType : featureTypes) {
+ String name = featureType.getName().getLocalPart();
+ FeatureTypeInfo ftinfo = factory.createFeatureType();
+ ftinfo.setEnabled(true);
+ ftinfo.setNativeName(name);
+ ftinfo.setName(name);
+ ftinfo.setNamespace(catalog.getDefaultNamespace());
+ List<AttributeTypeInfo> attributes = ftinfo.getAttributes();
+ for (AttributeDescriptor ad : featureType.getAttributeDescriptors()) {
+ AttributeTypeInfo att = factory.createAttribute();
+ att.setName(ad.getLocalName());
+ att.setBinding(ad.getType().getBinding());
+ attributes.add(att);
+ }
+
+ LayerInfo layer = cb.buildLayer((ResourceInfo) ftinfo);
+ ResourceInfo resource = layer.getResource();
+ resource.setSRS(KML_SRS);
+ resource.setNativeCRS(KML_CRS);
+ resource.setNativeBoundingBox(EMPTY_BOUNDS);
+ resource.setLatLonBoundingBox(EMPTY_BOUNDS);
+ resource.getMetadata().put("recalculate-bounds", Boolean.TRUE);
+
+ Map<Object, Object> userData = featureType.getUserData();
+ if (userData.containsKey("schemanames")) {
+ MetadataMap metadata = resource.getMetadata();
+ metadata.put("importschemanames", (Serializable) userData.get("schemanames"));
+ }
+
+ ImportItem item = new ImportItem(layer);
+ item.getMetadata().put(FeatureType.class, featureType);
+ result.add(item);
}
- LayerInfo layer = cb.buildLayer((ResourceInfo) ftinfo);
- ResourceInfo resource = layer.getResource();
- resource.setSRS(KML_SRS);
- resource.setNativeCRS(KML_CRS);
- resource.setNativeBoundingBox(EMPTY_BOUNDS);
- ImportItem item = new ImportItem(layer);
- item.getMetadata().put(FeatureType.class, featureType);
- return Collections.singletonList(item);
+ return Collections.unmodifiableList(result);
+ }
+
+ private String typeNameFromFile(File file) {
+ return FilenameUtils.getBaseName(file.getName());
}
}
Oops, something went wrong.

0 comments on commit ebbc560

Please sign in to comment.