Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

283 uploaded joined shapefile to geoserver has missing values #284

Merged
merged 5 commits into from
Mar 13, 2024
Merged

283 uploaded joined shapefile to geoserver has missing values #284

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b6173a9
Fixes #283 - changed join to create geopackage instead of shapefile
navarroc Mar 12, 2024
63553b6
Changed file upload to be the geoPkgFile
navarroc Mar 12, 2024
d7c4c86
pass in dataset id as the default join (#285)
longshuicy Mar 13, 2024
b644625
Temp fix layername (#286)
longshuicy Mar 13, 2024
ff310f7
Changed log info to debug and updated CHANGELOG
navarroc Mar 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions ...ce/src/main/java/edu/illinois/ncsa/incore/service/data/controllers/DatasetController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -938,8 +938,9 @@ public Dataset uploadFiles(@Parameter(name = "Dataset Id from data service", req
File joinedShapefile = null;
// todo: the join process for the network dataset should be added in here.
try {
joinedShapefile = FileUtils.joinShpTable(dataset, repository, true);
if (!GeoserverUtils.uploadShapefileToGeoserver(dataset.getId(), joinedShapefile)) {
//joinedShapefile = FileUtils.joinShpTable(dataset, repository, true);
geoPkgFile = FileUtils.joinShpTable(dataset, repository, true);
if (!GeoserverUtils.uploadGpkgToGeoserver(dataset.getId(), geoPkgFile)) {
logger.error("Fail to upload geopackage file");
throw new IncoreHTTPException(Response.Status.INTERNAL_SERVER_ERROR, "Fail to upload geopakcage file.");
}
Expand All @@ -952,7 +953,7 @@ public Dataset uploadFiles(@Parameter(name = "Dataset Id from data service", req
throw new IncoreHTTPException(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage());
}
// clean up
FileUtils.deleteTmpDir(joinedShapefile);
FileUtils.deleteTmpDir(geoPkgFile);
} else {
try {
if (format.equalsIgnoreCase(FileUtils.FORMAT_NETWORK)) {
Expand Down
9 changes: 6 additions & 3 deletions ...a-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeoserverRestApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ public boolean uploadToGeoserver(String store, File inFile, String inExt, Boolea
String fileName = FilenameUtils.getBaseName(inFile.getName());

// check if workspace exists
// TODO created never been used
boolean created = createWorkspace(this.geoserverUrl, GEOSERVER_WORKSPACE);
boolean published = false;

Expand Down Expand Up @@ -251,8 +252,10 @@ public Boolean uploadToGeoserverWithRenaming(String fileName, String store, File
String restUrl = this.geoserverUrl + "/rest";
String fileNameNoExt = fileName.split("\\.")[0];
int datastoreResponse = createDatastore(restUrl, GEOSERVER_WORKSPACE, store, inFile.getAbsolutePath(), fileForamt);
logger.info("Successfully created datastore for " + fileNameNoExt + " with response code " + datastoreResponse);
int layerResponse = createLayer(restUrl, GEOSERVER_WORKSPACE, store, store, fileNameNoExt);
if (datastoreResponse == 201 && layerResponse == 201) {
logger.info("Successfully created layer for " + fileNameNoExt + " with response code " + layerResponse);
if ((datastoreResponse == 201 || datastoreResponse == 200) && (layerResponse == 201 || layerResponse == 200)){
published = true;
}
} catch (IOException e) {
Expand Down Expand Up @@ -362,12 +365,12 @@ public int createLayer(String baseUrl, String workspaceName, String datastoreNam
*/
public int sendHttpRequest(String url, Map<String, String> headers, byte[] data,
Map<String, String> params, String method) throws IOException {
HttpURLConnection connection = createHttpConnection(url, method, headers);

if (params != null && !params.isEmpty()) {
url += getQueryString(params);
}

HttpURLConnection connection = createHttpConnection(url, method, headers);

try (OutputStream outputStream = connection.getOutputStream()) {
outputStream.write(data);
}
Expand Down
77 changes: 43 additions & 34 deletions ...data-service/src/main/java/edu/illinois/ncsa/incore/service/data/utils/GeotoolsUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException;
import edu.illinois.ncsa.incore.common.utils.GeoUtils;
import edu.illinois.ncsa.incore.service.data.models.Dataset;
import jakarta.ws.rs.core.Response;
import org.apache.commons.io.FilenameUtils;
import org.apache.log4j.Logger;
import org.geotools.coverage.grid.GridCoverage2D;
Expand All @@ -32,9 +33,6 @@
import org.geotools.feature.FeatureIterator;
import org.geotools.feature.simple.SimpleFeatureBuilder;
import org.geotools.feature.simple.SimpleFeatureTypeBuilder;
import org.geotools.feature.visitor.UniqueVisitor;
import org.geotools.filter.text.cql2.CQL;
import org.geotools.filter.text.cql2.CQLException;
import org.geotools.gce.arcgrid.ArcGridReader;
import org.geotools.gce.geotiff.GeoTiffFormat;
import org.geotools.gce.geotiff.GeoTiffWriteParams;
Expand All @@ -48,12 +46,10 @@
import org.opengis.feature.simple.SimpleFeature;
import org.opengis.feature.simple.SimpleFeatureType;
import org.opengis.feature.type.AttributeDescriptor;
import org.opengis.feature.type.GeometryDescriptor;
import org.opengis.filter.Filter;
import org.opengis.parameter.GeneralParameterValue;
import org.opengis.parameter.ParameterValueGroup;

import jakarta.ws.rs.core.Response;
import java.io.*;
import java.net.URL;
import java.nio.file.Files;
Expand Down Expand Up @@ -289,20 +285,21 @@ public static File joinTableShapefile(Dataset dataset, List<File> shpfiles, File
SimpleFeatureType csvFeatureType = createCsvFeatureType(csvFile.getPath(), shapeFileName);
SimpleFeatureCollection csvFeatures = createCsvFeatureFromCsvType(csvFile.getPath(), csvFeatureType);

SimpleFeatureCollection joinedFeatures = performInnerJoin(csvFeatures, shapefileSource, UNI_ID_CSV);
DefaultFeatureCollection newCollection = (DefaultFeatureCollection) performInnerJoin(csvFeatures, shapefileSource, UNI_ID_CSV, dataset.getId());

// to make an output to shapefile, use this
File outShapefile = outToShapefile(joinedFeatures, tempDir, outFileName, shapefileSource);
return outShapefile;
// File outShapefile = outToShapefile(joinedFeatures, tempDir, outFileName, shapefileSource);
// return outShapefile;

// to make an output to file, use this
// return outToFile(new File(tempDir + File.separator + outFileName), newSft, newCollection);

// to make an output to geopackage, use this
//return outToGpkgFile(new File(tempDir + File.separator + dataset.getId() + "." + FileUtils.EXTENSION_GEOPACKAGE), newCollection);
return outToGpkgFile(new File(tempDir + File.separator + dataset.getId() + "." + FileUtils.EXTENSION_GEOPACKAGE), newCollection);
}

public static File outToShapefile(SimpleFeatureCollection features, String outputDir, String outputFileName, SimpleFeatureSource inputShapefileSource) throws IOException {
public static File outToShapefile(SimpleFeatureCollection features, String outputDir, String outputFileName,
SimpleFeatureSource inputShapefileSource) throws IOException {
File shapefileOutputFile = new File(outputDir, outputFileName);
Map<String, Serializable> shapefileParams = Map.of(
"url", shapefileOutputFile.toURI().toURL(),
Expand All @@ -323,7 +320,8 @@ public static File outToShapefile(SimpleFeatureCollection features, String outpu
// write features to Shapefile
Transaction transaction = new DefaultTransaction("create");
String typeName = shapefileDataStore.getTypeNames()[0];
FeatureStore<SimpleFeatureType, SimpleFeature> featureStore = (FeatureStore<SimpleFeatureType, SimpleFeature>) shapefileDataStore.getFeatureSource(typeName);
FeatureStore<SimpleFeatureType, SimpleFeature> featureStore =
(FeatureStore<SimpleFeatureType, SimpleFeature>) shapefileDataStore.getFeatureSource(typeName);
featureStore.addFeatures(features);
transaction.commit();
transaction.close();
Expand Down Expand Up @@ -414,16 +412,14 @@ public static SimpleFeatureType createCsvFeatureType(String csvFilePath, String
}
}

public static SimpleFeatureCollection performInnerJoin(
SimpleFeatureCollection csvFeatures,
SimpleFeatureSource shapefileSource,
String commonFieldName) throws IOException {
public static SimpleFeatureCollection performInnerJoin(SimpleFeatureCollection csvFeatures, SimpleFeatureSource shapefileSource,
String commonFieldName, String datasetId) throws IOException {

// create an index for the CSV features
Map<Object, SimpleFeature> csvFeatureIndex = indexFeatures(csvFeatures, commonFieldName);

// create joined feature type
SimpleFeatureType joinedFeatureType = createJoinedFeatureType(csvFeatures.getSchema(), shapefileSource.getSchema());
SimpleFeatureType joinedFeatureType = createJoinedFeatureType(csvFeatures.getSchema(), shapefileSource.getSchema(), datasetId);

// create a DefaultFeatureCollection to store the joined features
DefaultFeatureCollection joinedFeatures = new DefaultFeatureCollection(null, joinedFeatureType);
Expand Down Expand Up @@ -458,36 +454,48 @@ public static Map<Object, SimpleFeature> indexFeatures(SimpleFeatureCollection f
return featureIndex;
}

public static SimpleFeature createJoinedFeature(SimpleFeature csvFeature, SimpleFeature shapefileFeature, SimpleFeatureType joinedFeatureType) {
public static SimpleFeature createJoinedFeature(SimpleFeature csvFeature, SimpleFeature shapefileFeature,
SimpleFeatureType joinedFeatureType) {
SimpleFeatureBuilder builder = new SimpleFeatureBuilder(joinedFeatureType);

// copy all attributes from CSV feature
for (int i = 0; i < csvFeature.getAttributeCount(); i++) {
builder.add(csvFeature.getAttribute(i));
}

List<String> attributes = new LinkedList<String>();
// copy all attributes from shapefile feature
for (int i = 0; i < shapefileFeature.getAttributeCount(); i++) {
String localName = shapefileFeature.getFeatureType().getDescriptor(i).getLocalName();
attributes.add(localName);
builder.add(shapefileFeature.getAttribute(i));
}

// copy all attributes from CSV feature that are not in the original shapefile
for (int i = 0; i < csvFeature.getAttributeCount(); i++) {
String localName = csvFeature.getFeatureType().getDescriptor(i).getLocalName();
if (!attributes.contains(localName)) {
builder.add(csvFeature.getAttribute(i));
}
}

SimpleFeature joinedFeature = builder.buildFeature(null);

return joinedFeature;
}

public static SimpleFeatureType createJoinedFeatureType(SimpleFeatureType csvType, SimpleFeatureType shapefileType) {
public static SimpleFeatureType createJoinedFeatureType(SimpleFeatureType csvType, SimpleFeatureType shapefileType, String datasetId) {
SimpleFeatureTypeBuilder builder = new SimpleFeatureTypeBuilder();
builder.setName("JoinedFeatureType");

addAttributesWithoutDuplicate(builder, csvType);
addAttributesWithoutDuplicate(builder, shapefileType, "guid");
GeometryDescriptor geometryDescriptor = shapefileType.getGeometryDescriptor();
if (geometryDescriptor != null) {
builder.setCRS(geometryDescriptor.getCoordinateReferenceSystem());
builder.add("geometry", geometryDescriptor.getType().getBinding());
builder.init(shapefileType);
if (!datasetId.equals("")) {
builder.setName(datasetId);
} else {
builder.setName("JoinedFeatureType");
}

addAttributesWithoutDuplicate(builder, csvType, "guid");
// addAttributesWithoutDuplicate(builder, shapefileType, "guid");
// GeometryDescriptor geometryDescriptor = shapefileType.getGeometryDescriptor();
// if (geometryDescriptor != null) {
// builder.setCRS(geometryDescriptor.getCoordinateReferenceSystem());
// builder.add("geometry", geometryDescriptor.getType().getBinding());
// }
longshuicy marked this conversation as resolved.
Show resolved Hide resolved

SimpleFeatureType builtFeatureType = builder.buildFeatureType();

return builtFeatureType;
Expand All @@ -497,7 +505,8 @@ public static void addAttributesWithoutDuplicate(SimpleFeatureTypeBuilder builde
addAttributesWithoutDuplicate(builder, sourceType, null);
}

public static void addAttributesWithoutDuplicate(SimpleFeatureTypeBuilder builder, SimpleFeatureType sourceType, String excludeAttribute) {
public static void addAttributesWithoutDuplicate(SimpleFeatureTypeBuilder builder, SimpleFeatureType sourceType,
String excludeAttribute) {
Set<String> addedAttributes = new HashSet<>();

for (int i = 0; i < sourceType.getAttributeCount(); i++) {
Expand Down Expand Up @@ -1127,7 +1136,7 @@ public static boolean doesGuidExist(SimpleFeatureCollection inputFeatures) {
* @return
* @throws IOException
*/
public static GeoUtils.gpkgValidationResult isGpkgFitToService(File inFile) throws IOException {
public static GeoUtils.gpkgValidationResult isGpkgFitToService(File inFile) throws IOException {
int output = 0;
try {
HashMap<String, Object> map = new HashMap<>();
Expand All @@ -1149,7 +1158,7 @@ public static GeoUtils.gpkgValidationResult isGpkgFitToService(File inFile) thr
String layerName = layerNames[0];
String fileName = inFile.getName().split("\\.")[0];
if (!layerName.equals(fileName)) {
return GeoUtils.gpkgValidationResult.NAME_MISMATCH;
return GeoUtils.gpkgValidationResult.NAME_MISMATCH;
}
} else if (layerNames.length == 0) {
return GeoUtils.gpkgValidationResult.RASTER_OR_NO_VECTOR_LAYER;
Expand Down
Loading