Skip to content

Commit

Permalink
feat: Add CSV Module, include basic standalone UI
Browse files Browse the repository at this point in the history
  • Loading branch information
rkorytkowski committed May 8, 2020
1 parent f13fead commit 6523511
Show file tree
Hide file tree
Showing 19 changed files with 410 additions and 24 deletions.
1 change: 0 additions & 1 deletion .gitignore
Expand Up @@ -76,4 +76,3 @@ Thumbs.db
**/bin
/ui/node
/ui/yarn-error.log

Expand Up @@ -18,18 +18,23 @@
import io.atlasmap.api.AtlasException;
import io.atlasmap.core.AtlasPath;
import io.atlasmap.core.AtlasUtil;
import io.atlasmap.csv.v2.CsvComplexType;
import io.atlasmap.csv.v2.CsvField;
import io.atlasmap.csv.v2.CsvFields;
import io.atlasmap.spi.AtlasFieldReader;
import io.atlasmap.spi.AtlasInternalSession;
import io.atlasmap.v2.AtlasModelFactory;
import io.atlasmap.v2.AuditStatus;
import io.atlasmap.v2.CollectionType;
import io.atlasmap.v2.Document;
import io.atlasmap.v2.Field;
import io.atlasmap.v2.FieldGroup;
import io.atlasmap.v2.FieldType;
import io.atlasmap.v2.Fields;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import sun.misc.IOUtils;

import java.io.BufferedInputStream;
import java.io.IOException;
Expand All @@ -55,8 +60,9 @@ public CsvFieldReader(CsvConfig csvConfig) {
public void setDocument(InputStream inputStream) {
if (inputStream != null && !inputStream.markSupported()) {
this.document = new BufferedInputStream(inputStream);
} else {
this.document = inputStream;
}
this.document = inputStream;
}

@Override
Expand Down Expand Up @@ -122,7 +128,12 @@ private Field readFields(CsvField field) throws AtlasException {
for (CSVRecord record: parser) {
if (i == fieldIndex) {
CsvField newField = CsvField.cloneOf(csvField);
String value = record.get(csvField.getColumn());
String value;
if (csvField.getColumn() != null) {
value = record.get(csvField.getColumn());
} else {
value = record.get(csvField.getName());
}
newField.setValue(value);
fields.add(newField);
break;
Expand Down Expand Up @@ -182,7 +193,7 @@ public Document readSchema() throws AtlasException {
throw new AtlasException(e);
}

List<Field> fields = new ArrayList<>();
List<CsvField> fields = new ArrayList<>();

if (csvConfig.isFirstRecordAsHeader()) {
int i = 0;
Expand All @@ -191,6 +202,7 @@ public Document readSchema() throws AtlasException {
field.setColumn(i);
field.setName(headerName);
field.setPath("/<>/" + headerName);
field.setFieldType(FieldType.STRING);
fields.add(field);
i++;
}
Expand All @@ -201,6 +213,7 @@ public Document readSchema() throws AtlasException {
field.setColumn(i);
field.setName(String.valueOf(i));
field.setPath("/<>/" + field.getName());
field.setFieldType(FieldType.STRING);
fields.add(field);
}
}
Expand All @@ -211,8 +224,18 @@ public Document readSchema() throws AtlasException {
throw new AtlasException(e);
}

CsvFields csvFields = new CsvFields();
csvFields.getCsvField().addAll(fields);

CsvComplexType csvComplexType = new CsvComplexType();
csvComplexType.setFieldType(FieldType.COMPLEX);
csvComplexType.setCollectionType(CollectionType.LIST);
csvComplexType.setPath("/<>");
csvComplexType.setName("");
csvComplexType.setCsvFields(csvFields);

Fields documentFields = new Fields();
documentFields.getField().addAll(fields);
documentFields.getField().add(csvComplexType);

Document document = new Document();
document.setFields(documentFields);
Expand Down
@@ -0,0 +1,153 @@
/**
* Copyright (C) 2017 Red Hat, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.atlasmap.csv.v2;

import com.fasterxml.jackson.annotation.JsonTypeInfo;

import java.io.Serializable;

@JsonTypeInfo(include = JsonTypeInfo.As.PROPERTY, use = JsonTypeInfo.Id.CLASS, property = "jsonType")
public class CsvComplexType extends CsvField implements Serializable {

private final static long serialVersionUID = 1L;

protected CsvFields csvFields;

protected String uri;

/**
* Gets the value of the csvFields property.
*
* @return
* possible object is
* {@link CsvFields }
*
*/
public CsvFields getCsvFields() {
return csvFields;
}

/**
* Sets the value of the csvFields property.
*
* @param value
* allowed object is
* {@link CsvFields }
*
*/
public void setCsvFields(CsvFields value) {
this.csvFields = value;
}

/**
* Gets the value of the uri property.
*
* @return
* possible object is
* {@link String }
*
*/
public String getUri() {
return uri;
}

/**
* Sets the value of the uri property.
*
* @param value
* allowed object is
* {@link String }
*
*/
public void setUri(String value) {
this.uri = value;
}

public boolean equals(Object object) {
if ((object == null)||(this.getClass()!= object.getClass())) {
return false;
}
if (this == object) {
return true;
}
if (!super.equals(object)) {
return false;
}
final CsvComplexType that = ((CsvComplexType) object);
{
CsvFields leftCsvFields;
leftCsvFields = this.getCsvFields();
CsvFields rightCsvFields;
rightCsvFields = that.getCsvFields();
if (this.csvFields!= null) {
if (that.csvFields!= null) {
if (!leftCsvFields.equals(rightCsvFields)) {
return false;
}
} else {
return false;
}
} else {
if (that.csvFields!= null) {
return false;
}
}
}
{
String leftUri;
leftUri = this.getUri();
String rightUri;
rightUri = that.getUri();
if (this.uri!= null) {
if (that.uri!= null) {
if (!leftUri.equals(rightUri)) {
return false;
}
} else {
return false;
}
} else {
if (that.uri!= null) {
return false;
}
}
}
return true;
}

public int hashCode() {
int currentHashCode = 1;
currentHashCode = ((currentHashCode* 31)+ super.hashCode());
{
currentHashCode = (currentHashCode* 31);
CsvFields theCsvFields;
theCsvFields = this.getCsvFields();
if (this.csvFields!= null) {
currentHashCode += theCsvFields.hashCode();
}
}
{
currentHashCode = (currentHashCode* 31);
String theUri;
theUri = this.getUri();
if (this.uri!= null) {
currentHashCode += theUri.hashCode();
}
}
return currentHashCode;
}

}
100 changes: 100 additions & 0 deletions lib/modules/csv/model/src/main/java/io/atlasmap/csv/v2/CsvFields.java
@@ -0,0 +1,100 @@
/**
* Copyright (C) 2017 Red Hat, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.atlasmap.csv.v2;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;

public class CsvFields implements Serializable {

private final static long serialVersionUID = 1L;

protected List<CsvField> csvField;

/**
* Gets the value of the csvField property.
*
* <p>
* This accessor method returns a reference to the live list,
* not a snapshot. Therefore any modification you make to the
* returned list will be present inside the JAXB object.
* This is why there is not a <CODE>set</CODE> method for the csvField property.
*
* <p>
* For example, to add a new item, do as follows:
* <pre>
* getCsvField().add(newItem);
* </pre>
*
*
* <p>
* Objects of the following type(s) are allowed in the list
* {@link CsvField }
*
*
*/
public List<CsvField> getCsvField() {
if (csvField == null) {
csvField = new ArrayList<CsvField>();
}
return this.csvField;
}

public boolean equals(Object object) {
if ((object == null)||(this.getClass()!= object.getClass())) {
return false;
}
if (this == object) {
return true;
}
final CsvFields that = ((CsvFields) object);
{
List<CsvField> leftCsvField;
leftCsvField = (((this.csvField != null)&&(!this.csvField.isEmpty()))?this.getCsvField():null);
List<CsvField> rightCsvField;
rightCsvField = (((that.csvField!= null)&&(!that.csvField.isEmpty()))?that.getCsvField():null);
if ((this.csvField != null)&&(!this.csvField.isEmpty())) {
if ((that.csvField!= null)&&(!that.csvField.isEmpty())) {
if (!leftCsvField.equals(rightCsvField)) {
return false;
}
} else {
return false;
}
} else {
if ((that.csvField!= null)&&(!that.csvField.isEmpty())) {
return false;
}
}
}
return true;
}

public int hashCode() {
int currentHashCode = 1;
{
currentHashCode = (currentHashCode* 31);
List<CsvField> theCsvField;
theCsvField = (((this.csvField != null)&&(!this.csvField.isEmpty()))?this.getCsvField():null);
if ((this.csvField != null)&&(!this.csvField.isEmpty())) {
currentHashCode += theCsvField.hashCode();
}
}
return currentHashCode;
}

}
Expand Up @@ -27,12 +27,12 @@ public class CsvInspectionResponse implements Serializable {

private final static long serialVersionUID = 1L;

private Document document;
private Document csvDocument;
private String errorMessage;
private long executionTime;

public Document getDocument() {
return document;
public Document getCsvDocument() {
return csvDocument;
}

public String getErrorMessage() {
Expand All @@ -43,8 +43,8 @@ public long getExecutionTime() {
return executionTime;
}

public void setDocument(Document document) {
this.document = document;
public void setCsvDocument(Document csvDocument) {
this.csvDocument = csvDocument;
}

public void setErrorMessage(String message) {
Expand Down
Expand Up @@ -121,8 +121,9 @@ public Response inspect(InputStream request, @QueryParam("format") String format

CsvFieldReader csvFieldReader = new CsvFieldReader(csvConfig);
csvFieldReader.setDocument(request);

Document document = csvFieldReader.readSchema();
response.setDocument(document);
response.setCsvDocument(document);
request.close();
} catch (Exception e) {
LOG.error("Error inspecting CSV: " + e.getMessage(), e);
Expand Down

0 comments on commit 6523511

Please sign in to comment.