Skip to content

Commit

Permalink
Make TableSchema errors more understandable + add diff
Browse files Browse the repository at this point in the history
  • Loading branch information
@Matthew-Kulich
Matthew-Kulich committed Oct 7, 2022
1 parent 3ddf392 commit 5fd4519
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 36 deletions.
23 changes: 13 additions & 10 deletions s-pipes-modules/module-tabular/src/main/java/cz/cvut/spipes/modules/TabularModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,9 +128,9 @@ ExecutionContext executeSelf() {
List<Statement> rowStatements = new ArrayList<>();

CsvPreference csvPreference = new CsvPreference.Builder(
quoteCharacter,
delimiter,
"\\n").build();
quoteCharacter,
delimiter,
"\\n").build();

try{
ICsvListReader listReader = new CsvListReader(getReader(), csvPreference);
Expand All @@ -151,6 +151,7 @@ ExecutionContext executeSelf() {
}else if (hasInputSchema) {
header = getHeaderFromSchema(inputModel, header, true);
}
tableSchema.setHeader(header);

em = JopaPersistenceUtils.getEntityManager("cz.cvut.spipes.modules.model", outputModel);
em.getTransaction().begin();
Expand All @@ -161,13 +162,12 @@ ExecutionContext executeSelf() {
String columnName = normalize(columnTitle);
boolean isDuplicate = !columnNames.add(columnName);

Column schemaColumn = hasInputSchema && tableSchema.getColumn(columnName) != null
? tableSchema.getColumn(columnName)
: new Column(columnName, columnTitle);
Column schemaColumn = new Column(columnName, columnTitle);
outputColumns.add(schemaColumn);

tableSchema.setAboutUrl(schemaColumn, sourceResource.getUri());
schemaColumn.setProperty(dataPrefix, sourceResource.getUri());
schemaColumn.setProperty(dataPrefix, sourceResource.getUri(),
hasInputSchema ? tableSchema.getColumn(columnName) : null);
schemaColumn.setTitle(columnTitle);
if(isDuplicate) throwNotUniqueException(schemaColumn,columnTitle, columnName);
}
Expand Down Expand Up @@ -220,6 +220,9 @@ ExecutionContext executeSelf() {

tableSchema.adjustProperties(hasInputSchema, outputColumns, sourceResource.getUri());
em.persist(tableGroup);

tableSchema.setColumnsSet(new HashSet<>(outputColumns));
em.merge(tableSchema);
em.getTransaction().commit();
tableSchema.addColumnsList(em, outputColumns);

Expand Down Expand Up @@ -275,8 +278,8 @@ private TableSchema getTableSchema(EntityManager em) {
private void throwNotUniqueException(Column column, String columnTitle, String columnName) {
throw new ResourceNotUniqueException(
String.format("Unable to create value of property %s due to collision. " +
"Both column titles '%s' and '%s' are normalized to '%s' " +
"and thus would refer to the same property url <%s>.",
"Both column titles '%s' and '%s' are normalized to '%s' " +
"and thus would refer to the same property url <%s>.",
CSVW.propertyUrl,
columnTitle,
column.getTitle(),
Expand All @@ -301,7 +304,7 @@ public void loadConfiguration() {
dataPrefix = getEffectiveValue(P_DATE_PREFIX).asLiteral().toString();
sourceResource = getResourceByUri(getEffectiveValue(P_SOURCE_RESOURCE_URI).asLiteral().toString());
outputMode = Mode.fromResource(
getPropertyValue(P_OUTPUT_MODE, Mode.STANDARD.getResource())
getPropertyValue(P_OUTPUT_MODE, Mode.STANDARD.getResource())
);
}

Expand Down
27 changes: 19 additions & 8 deletions s-pipes-modules/module-tabular/src/main/java/cz/cvut/spipes/modules/model/Column.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ public class Column extends AbstractEntity {

public Column() {}

public Column(String name) {
this.name = name;
}

public Column(String name, String title) {
this.name = name;
this.title = title;
Expand Down Expand Up @@ -133,19 +137,26 @@ public String getProperty() {
return property;
}

public void setProperty(String dataPrefix, String sourceResourceUri) throws UnsupportedEncodingException {
String propertyValue = getPropertyUrl(dataPrefix, sourceResourceUri);
public void setProperty(String dataPrefix, String sourceResourceUri, Column schemaColumn) throws UnsupportedEncodingException {
String propertyValue = getPropertyUrl(dataPrefix, sourceResourceUri, schemaColumn);
tabularModuleUtils.setVariable(this.property, propertyValue, value -> this.property = value, "property");
tabularModuleUtils.setVariable(this.propertyUrl, propertyValue, value -> this.propertyUrl = value, "propertyUrl");
}

private String getPropertyUrl(String dataPrefix, String sourceResourceUri)
public void setProperty(String property){
this.property = property;
this.propertyUrl = property;
}

private String getPropertyUrl(String dataPrefix, String sourceResourceUri, Column schemaColumn)
throws UnsupportedEncodingException {
if (getPropertyUrl() != null) {
return getPropertyUrl();
}
if (getProperty() != null) {
return getProperty();
if (schemaColumn != null){
if (schemaColumn.getPropertyUrl() != null){
return schemaColumn.getPropertyUrl();
}
if (schemaColumn.getProperty() != null){
return schemaColumn.getProperty();
}
}
if (dataPrefix != null && !dataPrefix.isEmpty()) {
return dataPrefix + URLEncoder.encode(name, "UTF-8");
Expand Down
65 changes: 47 additions & 18 deletions s-pipes-modules/module-tabular/src/main/java/cz/cvut/spipes/modules/model/TableSchema.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ public class TableSchema extends AbstractEntity {

private final transient TabularModuleUtils tabularModuleUtils = new TabularModuleUtils();

private transient String[] header;

public String getAboutUrl() {
return aboutUrl;
}
Expand Down Expand Up @@ -98,29 +100,55 @@ public List<Column> sortColumns(List<String> orderList){

public void adjustProperties(boolean hasInputSchema, List<Column> outputColumns, String sourceResourceUri) {
if (hasInputSchema){
if(getColumnsSet().size() > outputColumns.size()) {
throwExtraColumnsError(outputColumns);
if (columnsSet.isEmpty()) logError("Input schema has no columns.");
if (!columnsSet.isEmpty()){
checkColumnsConsistency(outputColumns);
}
setColumnsSet(new HashSet<>());
setAboutUrl(sourceResourceUri + "#row-{_row}");
getColumnsSet().forEach(column -> column.setUri(null));
setUri(null);
}else{
setColumnsSet(new HashSet<>(outputColumns));
}
}

private void throwExtraColumnsError(List<Column> outputColumns) {
StringBuilder errorMessage =
new StringBuilder("There is an additional column in retrieved input data schema compared to expected one");
private void checkColumnsConsistency(List<Column> outputColumns) {
StringBuilder errorMessage = new StringBuilder();
String missingColumnMessage = "There is missing column in retrieved input data compared to expected one.\n" +
"Missing columns: ";
String additionalColumnMessage = "\nThere is an additional column in retrieved input data schema compared" +
" to expected one: \nExtra columns:\t ";

for (Column column : getColumnsSet()) {
if (outputColumns.stream().noneMatch(outputColumn -> outputColumn.getName().equals(column.getName()))) {
errorMessage
.append("\n")
.append(String.format("Column with name `%s` is extra.", column.getName()));
errorMessage.append(throwColumnsError(outputColumns, new ArrayList<>(getColumnsSet()), missingColumnMessage));
errorMessage.append(throwColumnsError(new ArrayList<>(getColumnsSet()), outputColumns, additionalColumnMessage));

if (errorMessage.length() > 0) {
errorMessage.append(addSchemaDiff());
logError(errorMessage.toString());
}
}

private StringBuilder throwColumnsError(List<Column> outputColumns, List<Column> columnsList, String message) {
StringBuilder errorMessage = new StringBuilder();

for (Column col: outputColumns){
if (columnsList.stream().noneMatch(column -> column.getName().equals(col.getName()))){
errorMessage.append("'").append(col.getName()).append(String.format("%-20s", "'\n"));
}
}
logError(errorMessage.toString());

if (errorMessage.length() > 0) errorMessage.insert(0, message);
return errorMessage;
}

private String addSchemaDiff() {
StringBuilder errorMessage = new StringBuilder("\nActual: \n")
.append("| ")
.append(String.join(" | ", header))
.append(" |\nExpected: \n| ");

columnsSet.stream()
.map(Column::getName)
.forEach(name -> errorMessage.append(name.replace("_"," ")).append(" | "));
return errorMessage.toString();
}

public void addColumnsList(EntityManager em, List<Column> outputColumns) {
Expand Down Expand Up @@ -163,10 +191,7 @@ public Column getColumn(String columnName) {
}
}

String errorMessage = String.format("There is missing column in retrieved input schema compared to expected one" +
"\n Column `%s` does not exist in input schema.", columnName);
logError(errorMessage);
return null;
return new Column(columnName);
}

private void logError(String msg) {
Expand All @@ -184,4 +209,8 @@ public String createAboutUrl(int rowNumber) {
);
return columnAboutUrlStr;
}

public void setHeader(String[] header) {
this.header = header;
}
}

0 comments on commit 5fd4519

Please sign in to comment.