Implemented Automatically Generate Literal Types feature and fixed bug

karma-app/build

@@ -10,6 +10,7 @@
 # 6. Navigates to the electron app folder and copies the tomcat folder to app/app directory.
 # 7. Runs the release script there to build karma app for linux, windows and mac.
 ########################################
+set -e
 
 CURRENT=`pwd`
 rm -rf ./target
@@ -18,6 +19,8 @@ mkdir target
 cd ..
 mvn clean install -Dmaven.test.skip=true
 cd karma-web/
+
+set +e
 rm src/main/webapp/publish/CSV/*.csv
 rm src/main/webapp/publish/AVRO/*.avro
 rm src/main/webapp/publish/JSON/*.json
@@ -29,6 +32,7 @@ rm src/main/webapp/publish/RDF/*.rdf
 rm src/main/webapp/publish/RDF/*.ttl
 rm src/main/webapp/publish/RDF/*.txt
 rm src/main/webapp/publish/REPORT/*.md
+set -e
 
 mvn package -Pshaded
 cp target/karma-web*.war $CURRENT/target/ROOT.war
@@ -45,9 +49,9 @@ mv karma-app-deps-master/*.tar.gz .
 rm -rf karma-app-deps-master master.zip
 
 # download tomcat binary
-wget http://apache.cs.utah.edu/tomcat/tomcat-8/v8.5.29/bin/apache-tomcat-8.5.29.zip
-unzip apache-tomcat-8.5.29.zip
-rm apache-tomcat-8.5.29.zip
+wget http://mirror.cc.columbia.edu/pub/software/apache/tomcat/tomcat-8/v8.5.30/bin/apache-tomcat-8.5.30.zip
+unzip apache-tomcat-*.zip
+rm apache-tomcat-*.zip
 
 mv apache-tomcat* tomcat
 rm -rf tomcat/webapps/*
@@ -84,11 +88,14 @@ rm -rf jre*
 cp -rf Karma-darwin* $CURRENT/target/Karma-Mac
 cp -rf Karma-linux* $CURRENT/target/Karma-Linux
 cp -rf Karma-win* $CURRENT/target/Karma-Windows
+
+set +e
 rm -rf Karma-*
 
 cd $CURRENT/target
 rm *.tar.gz
 
+set -e
 #put our license
 for build_folder in `ls |grep Karma`
 do

karma-app/desktop/package.json

@@ -30,7 +30,7 @@
   },
   "devDependencies": {
     "chai": "^3.5.0",
-    "electron": "1.8.4",
+    "electron": "2.0.0-beta.7",
     "electron-builder": "^5.12.1",
     "electron-mocha": "^3.0.0",
     "fs-jetpack": "^0.9.0",

karma-commands/commands-alignment/src/main/java/edu/isi/karma/controller/command/alignment/GetPropertiesCommand.java

@@ -44,7 +44,7 @@ public enum INTERNAL_PROP_RANGE {
 	}
 
 	private enum JsonKeys {
-		updateType, label, id, properties, uri, type, rdfsLabel
+		updateType, label, id, properties, uri, type, rdfsLabel, literaltype,
 	}
 
 	private String classURI, domainURI, rangeURI, linkId;
@@ -86,7 +86,7 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException {
 		final OntologyManager ontMgr = workspace.getOntologyManager();
 		Set<LabeledLink> properties = new HashSet<>();
 
-		logger.debug("GetPropertiesCommand:" + propertiesRange + ":" + classURI + "," + domainURI + ", " +  rangeURI);
+//		logger.z("GetPropertiesCommand:" + propertiesRange + ":" + classURI + "," + domainURI + ", " +  rangeURI);
 
 		if (propertiesRange == INTERNAL_PROP_RANGE.allObjectProperties) {
 			HashMap<String, Label> linkList = ontMgr.getObjectProperties();
@@ -164,14 +164,15 @@ public void generateJson(String prefix, PrintWriter pw,
 						Label linkLabel = link.getLabel();
 						String edgeLabelStr = linkLabel.getDisplayName();
 						JSONObject edgeObj = new JSONObject();
-						if (linkLabel.getUri() !=null && linkLabel.getNs() != null 
+						if (linkLabel.getUri() !=null && linkLabel.getNs() != null
 								&& linkLabel.getUri().equalsIgnoreCase(linkLabel.getNs())) {
 							edgeLabelStr = linkLabel.getUri();
 						}
-
+
+
 						edgeObj.put(JsonKeys.label.name(), edgeLabelStr);
 						edgeObj.put(JsonKeys.rdfsLabel.name(), linkLabel.getRdfsLabel());
-							
+						edgeObj.put(JsonKeys.literaltype.name(), linkLabel.getRdfsRange());
 						edgeObj.put(JsonKeys.uri.name(), linkLabel.getUri());
 						edgeObj.put(JsonKeys.id.name(), link.getId());
 

karma-commands/commands-alignment/src/main/java/edu/isi/karma/controller/command/alignment/SetMetaPropertyCommand.java

@@ -197,7 +197,7 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException {
 			newType = new SemanticType(hNodeId,
 					ClassInstanceLink.getFixedLabel(), classNode.getLabel(),
 					classNode.getId(),
-					false,
+					false, false,
 					SemanticType.Origin.User, 1.0);
 		} else if (metaPropertyName
 				.equals(METAPROPERTY_NAME.isSpecializationForEdge)) {
@@ -238,7 +238,7 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException {
 						DataPropertyOfColumnLink.getFixedLabel(),
 						classInstanceNode.getLabel(), 
 						classInstanceNode.getId(),
-						false,
+						false, false,
 						SemanticType.Origin.User,
 						1.0);
 			} else if (propertyLink instanceof ObjectPropertyLink) {
@@ -252,7 +252,7 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException {
 						ObjectPropertySpecializationLink.getFixedLabel(),
 						classInstanceNode.getLabel(), 
 						classInstanceNode.getId(),
-						false,
+						false, false,
 						SemanticType.Origin.User,
 						1.0);
 			}
@@ -278,7 +278,7 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException {
 
 			// Create the semantic type object
 			newType = new SemanticType(hNodeId,
-					ColumnSubClassLink.getFixedLabel(), classNode.getLabel(), classNode.getId(), false,
+					ColumnSubClassLink.getFixedLabel(), classNode.getLabel(), classNode.getId(), false, false,
 					SemanticType.Origin.User, 1.0);
 		}
 

karma-commands/commands-alignment/src/main/java/edu/isi/karma/controller/command/alignment/SetSemanticTypeCommand.java

@@ -21,6 +21,7 @@
 package edu.isi.karma.controller.command.alignment;
 
 import java.util.*;
+import java.lang.System;
 
 import org.jgrapht.graph.DirectedWeightedMultigraph;
 import org.json.JSONArray;
@@ -86,6 +87,8 @@ protected SetSemanticTypeCommand(String id, String model, String worksheetId, St
 		this.language = language;
 		this.hasProvenanceType = false;
 		addTag(CommandTag.SemanticType);
+
+//		System.out.println("types array" + this.typesArr.toString());
 	}
 
 	@Override
@@ -172,6 +175,7 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException {
 			}
 		}
 		this.typesArr = newTypesArr;
+//		System.out.println("types array 2" + this.typesArr.toString());
 
 		/*** Preprocess provenance types ***/
 		JSONArray provTypes = new JSONArray();
@@ -217,6 +221,9 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException {
 		JSONArray inputParams = new JSONArray(this.getInputParameterJson());
 		HistoryJsonUtil.setArgumentValue(Arguments.SemanticTypesArray.name(), typesArr, inputParams);
 		this.setInputParameterJson(inputParams.toString());
+
+
+//		System.out.println("input params" + inputParams.toString());
 
 		/*** Add the appropriate nodes and links in alignment graph ***/
 		ArrayList<SemanticType> typesList = new ArrayList<>();
@@ -259,6 +266,10 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException {
 				boolean isProvenance = false;
 				if(type.has(ClientJsonKeys.isProvenance.name()))
 					isProvenance = type.getBoolean(ClientJsonKeys.isProvenance.name());
+
+				boolean isPrimary = false;
+				if(type.has(ClientJsonKeys.isPrimary.name()))
+					isPrimary = type.getBoolean(ClientJsonKeys.isPrimary.name());
 
 				Node source = alignment.getNodeById(sourceId);
 				if (source == null) {
@@ -284,7 +295,7 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException {
 				}
 				newLink = alignment.addDataPropertyLink(source, columnNode, linkLabel, isProvenance);
 				SemanticType newType = new SemanticType(hNodeId, linkLabel, source.getLabel(), source.getId(),
-											isProvenance,
+											isProvenance, isPrimary,
 											SemanticType.Origin.User, 1.0);
 
 				List<SemanticType> userSemanticTypes = columnNode.getUserSemanticTypes();

karma-commands/commands-alignment/src/main/java/edu/isi/karma/controller/command/alignment/SuggestAutoModelCommand.java

@@ -128,7 +128,7 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException {
 				alignment.addDataPropertyLink(classNode, columnNode, propertyLabel, false);
 
 				// Create a semantic type object
-				SemanticType type = new SemanticType(hNode.getId(), propertyLabel, internalNodeLabel, classNode.getId(), false, SemanticType.Origin.User, 1.0);
+				SemanticType type = new SemanticType(hNode.getId(), propertyLabel, internalNodeLabel, classNode.getId(), false, false, SemanticType.Origin.User, 1.0);
 				worksheet.getSemanticTypes().addType(type);
 
 				List<SemanticType> userSemanticTypes = columnNode.getUserSemanticTypes();

karma-commands/commands-alignment/src/main/java/edu/isi/karma/controller/command/alignment/SuggestAutoModelCommandFactory.java

@@ -122,7 +122,7 @@ public Command createCommand(JSONArray inputJson, String model, Workspace worksp
 
 			if (typeName != null) {
 				type = new SemanticType(hNodeId, typeName, domainName, domainId, 
-						false, Origin.User, 1.00);
+						false, false, Origin.User, 1.00);
 				worksheet.getSemanticTypes().addType(type);
 			}
 		}

karma-commands/commands-alignment/src/main/java/edu/isi/karma/controller/command/alignment/SuggestModelCommandFactory.java

@@ -84,7 +84,7 @@ public Command createCommand(JSONArray inputJson, String model, Workspace worksp
 				domainName = ontMgr.getUriLabel(domain);
 
 			if(typeName != null) {
-				type = new SemanticType(hNodeId, typeName, domainName, domainId, false, Origin.User, 1.00);
+				type = new SemanticType(hNodeId, typeName, domainName, domainId, false, false, Origin.User, 1.00);
 				worksheet.getSemanticTypes().addType(type);
 			}
 		}

karma-commands/commands-python/src/main/java/edu/isi/karma/controller/command/transformation/MutatingPythonTransformationCommand.java

@@ -112,7 +112,7 @@ protected UpdateContainer applyPythonTransformation(Workspace workspace,
 		/** Add the alignment update **/
 		c.append(computeAlignmentAndSemanticTypesAndCreateUpdates(workspace));
 
-		c.add(new InfoUpdate("Transformation complete"));
+		c.add(new InfoUpdate("Transformation updated"));
 		return c;
 	}
 

karma-common/src/main/java/edu/isi/karma/controller/history/WorksheetCommandHistoryExecutor.java

@@ -121,6 +121,10 @@ private UpdateContainer executeCommand(JSONObject commObject, boolean saveToHist
 						} catch(Exception e) {
 							logger.error("Error executing command: "+ commandName + ". Please notify this error. \nInputs:" + inputParamArr, e);
 							//make these InfoUpdates so that the UI can still process the rest of the model
+							if(commandName.equals("SetSemanticTypeCommand")) {
+								System.out.println("commandName equal to SetSemanticTypeCommand");
+								return new UpdateContainer(new TrivialErrorUpdate("Error executing command " + commandName + " from history, since you may have deleted the related column."));
+							}
 							return new UpdateContainer(new TrivialErrorUpdate("Error executing command " + commandName + " from history"));
 						}
 					}

karma-common/src/main/java/edu/isi/karma/controller/update/SemanticTypesUpdate.java

@@ -25,6 +25,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
+import java.lang.System;
 
 import org.json.JSONException;
 import org.json.JSONStringer;
@@ -106,7 +107,7 @@ public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) {
 			}
 			writer.endArray();
 			writer.endObject();
-
+//			System.out.println("writer " + writer.toString());
 			pw.print(writer.toString());
 		} catch (JSONException e) {
 			logger.error("Error occured while writing to JSON!", e);
@@ -131,7 +132,8 @@ private void writeType(SemanticType type, JSONWriter writer, Map<String, ColumnN
 					.key(JsonKeys.ConfidenceLevel.name())
 					.value(type.getConfidenceLevel().name())
 					.key(JsonKeys.isPrimary.name())
-					.value(true);
+					.value(type.isPrimary());
+
 
 			// Add the RDF literal type to show in the text box
 			String rdfLiteralType = alignmentColumnNode.getRdfLiteralType() == null? "" : 
@@ -178,6 +180,7 @@ private void writeType(SemanticType type, JSONWriter writer, Map<String, ColumnN
 
 
 			writer.endObject();
+
 		}
 	}
 

karma-common/src/main/java/edu/isi/karma/kr2rml/planning/TriplesMapPlanExecutor.java

@@ -64,7 +64,8 @@ public ErrorReport execute(final TriplesMapPlan plan)
 				List<Future<Boolean>> results = service.invokeAll(plan.workers);
 				for(Future<Boolean> result : results)
 				{
-					result.get(3, TimeUnit.MINUTES);
+
+					result.get(10, TimeUnit.MINUTES);
 				}
 			}
 			else
@@ -81,7 +82,7 @@ public Boolean call() throws Exception {
 						}
 
 					}); 
-				result.get(1, TimeUnit.MINUTES);
+				result.get(10, TimeUnit.MINUTES);
 			}
 
 		} catch (Exception e) {

karma-common/src/main/java/edu/isi/karma/modeling/Uris.java

@@ -36,6 +36,7 @@ public abstract class Uris {
 	public static final String RDFS_CLASS_URI = Namespaces.RDFS + "Class";
 	public static final String RDFS_LABEL_URI = Namespaces.RDFS + "label";
 	public static final String RDFS_COMMENT_URI = Namespaces.RDFS + "comment";
+	public static final String RDFS_RANGE_URI = Namespaces.RDFS + "range";
 	public static final String RDF_VALUE_URI = Namespaces.RDF + "value";
 
 	public static final String OWL_SAMEAS_URI = Namespaces.OWL + "sameAs";

karma-common/src/main/java/edu/isi/karma/modeling/alignment/Alignment.java

@@ -1110,7 +1110,7 @@ public void updateAlignment(SemanticModel model, List<SemanticType> semanticType
 			if (target instanceof ColumnNode) {
 				SemanticType st = new SemanticType(((ColumnNode)target).getHNodeId(), 
 						newLink.getLabel(), source.getLabel(), source.getId(),
-						false,
+						false, false,
 						SemanticType.Origin.User, 1.0);
 				semanticTypes.add(st);
 			}

karma-common/src/main/java/edu/isi/karma/modeling/alignment/GraphUtil.java

@@ -968,7 +968,7 @@ private static SemanticType readSemanticType(JsonReader reader) throws IOExcepti
 		}
     	reader.endObject();
 
-    	SemanticType semanticType = new SemanticType(hNodeId, type, domain, null, false, origin, confidenceScore);
+    	SemanticType semanticType = new SemanticType(hNodeId, type, domain, null, false, false, origin, confidenceScore);
     	return semanticType;	
     }
 

karma-common/src/main/java/edu/isi/karma/modeling/alignment/SemanticModel.java

@@ -250,7 +250,7 @@ private void setUserTypesForColumnNodes() {
 				for (LabeledLink link : incomingLinks) {
 					Node domain = link.getSource();
 					SemanticType st = new SemanticType(cn.getHNodeId(), link.getLabel(), 
-							domain.getLabel(), domain.getId(), false, 
+							domain.getLabel(), domain.getId(), false, false,
 							Origin.User, 1.0);
 					cn.assignUserType(st);
 				}

karma-common/src/main/java/edu/isi/karma/modeling/ontology/OntologyCache.java

@@ -571,12 +571,20 @@ private void loadProperties() {
 //			if (p.isAnnotationProperty())
 //				continue;
 
+
+
 			if (!properties.containsKey(p.getURI()))
 				properties.put(p.getURI(), ontHandler.getResourceLabel(p));
-
+			//datatype property put ? in hashmap
 			if (p.isDatatypeProperty() || !p.isObjectProperty())
 			{
 				if (!dataProperties.containsKey(p.getURI()))
+//					if (p.getRange() != null){
+//						logger.error("data type pro " + p);
+//						logger.error("get range " + p.getRange());
+//						logger.error("dp " + ontHandler.getResourceLabel(p));
+//					}
+
 					dataProperties.put(p.getURI(), ontHandler.getResourceLabel(p));
 			}