schema definition in config json part 7 (model id 103)

examples/amazon-google/config.json

@@ -47,16 +47,7 @@
 				"delimiter": ",",
 				"header":true					
 			},
-			"schema": 
-				"{\"type\" : \"struct\",
-				\"fields\" : [ 
-					{\"name\":\"id\", \"type\":\"string\", \"nullable\":false}, 
-					{\"name\":\"title\", \"type\":\"string\", \"nullable\":true},
-					{\"name\":\"description\", \"type\":\"string\", \"nullable\":true},
-					{\"name\":\"manufacturer\",\"type\":\"string\",\"nullable\":true} ,
-					{\"name\":\"price\", \"type\":\"double\", \"nullable\":true}
-				]
-			}"
+			"schema": "id string, title string, description string, manufacturer string, price double"
 		},
 		{
 			"name":"google", 
@@ -66,16 +57,7 @@
 				"delimiter": ",",
 				"header":true					
 			},
-			"schema": 
-				"{\"type\" : \"struct\",
-				\"fields\" : [ 
-					{\"name\":\"id\", \"type\":\"string\", \"nullable\":false}, 
-					{\"name\":\"title\", \"type\":\"string\", \"nullable\":true},
-					{\"name\":\"description\", \"type\":\"string\", \"nullable\":true},
-					{\"name\":\"manufacturer\",\"type\":\"string\",\"nullable\":true} ,
-					{\"name\":\"price\", \"type\":\"double\", \"nullable\":true}
-				]
-			}"
+			"schema": "id string, title string, description string, manufacturer string, price double"
 		}
 		],
 		"labelDataSampleSize" : 0.4,

examples/amazon-google/configWithStopWords.json

@@ -48,16 +48,7 @@
 				"delimiter": ",",
 				"header":true					
 			},
-			"schema": 
-				"{\"type\" : \"struct\",
-				\"fields\" : [ 
-					{\"name\":\"id\", \"type\":\"string\", \"nullable\":false}, 
-					{\"name\":\"title\", \"type\":\"string\", \"nullable\":true},
-					{\"name\":\"description\", \"type\":\"string\", \"nullable\":true},
-					{\"name\":\"manufacturer\",\"type\":\"string\",\"nullable\":true} ,
-					{\"name\":\"price\", \"type\":\"double\", \"nullable\":true}
-				]
-			}"
+			"schema": "id string, title string, description string, manufacturer string, price double"
 		},
 		{
 			"name":"google", 
@@ -67,16 +58,7 @@
 				"delimiter": ",",
 				"header":true					
 			},
-			"schema": 
-				"{\"type\" : \"struct\",
-				\"fields\" : [ 
-					{\"name\":\"id\", \"type\":\"string\", \"nullable\":false}, 
-					{\"name\":\"title\", \"type\":\"string\", \"nullable\":true},
-					{\"name\":\"description\", \"type\":\"string\", \"nullable\":true},
-					{\"name\":\"manufacturer\",\"type\":\"string\",\"nullable\":true} ,
-					{\"name\":\"price\", \"type\":\"double\", \"nullable\":true}
-				]
-			}"
+			"schema": "id string, title string, description string, manufacturer string, price double"
 		}
 		],
 		"labelDataSampleSize" : 0.4,

models/103/model/classifier/best.model/bestModel/metadata/part-00000

@@ -1 +1 @@
-{"class":"org.apache.spark.ml.PipelineModel","timestamp":1658797719173,"sparkVersion":"3.1.2","uid":"pipeline_46ab3b798dc1","paramMap":{"stageUids":["vecAssembler_cb42757e866f","poly_1c7b648c7b13","logreg_a2f6b7295985"]},"defaultParamMap":{}}
+{"class":"org.apache.spark.ml.PipelineModel","timestamp":1680102281210,"sparkVersion":"3.1.2","uid":"pipeline_78b89ec13f6a","paramMap":{"stageUids":["vecAssembler_e81b419dfd39","poly_d53366c8ab44","logreg_3ad85ac653f2"]},"defaultParamMap":{}}

models/103/model/classifier/best.model/estimator/metadata/part-00000

@@ -1 +1 @@
-{"class":"org.apache.spark.ml.Pipeline","timestamp":1658797718738,"sparkVersion":"3.1.2","uid":"pipeline_46ab3b798dc1","paramMap":{"stageUids":["vecAssembler_cb42757e866f","poly_1c7b648c7b13","logreg_a2f6b7295985"]},"defaultParamMap":{}}
+{"class":"org.apache.spark.ml.Pipeline","timestamp":1680102280638,"sparkVersion":"3.1.2","uid":"pipeline_78b89ec13f6a","paramMap":{"stageUids":["vecAssembler_e81b419dfd39","poly_d53366c8ab44","logreg_3ad85ac653f2"]},"defaultParamMap":{}}

models/103/model/classifier/best.model/evaluator/metadata/part-00000

@@ -1 +1 @@
-{"class":"org.apache.spark.ml.evaluation.BinaryClassificationEvaluator","timestamp":1658797718627,"sparkVersion":"3.1.2","uid":"binEval_29fc509a0ee2","paramMap":{"labelCol":"z_isMatch"},"defaultParamMap":{"rawPredictionCol":"rawPrediction","numBins":1000,"metricName":"areaUnderROC","labelCol":"label"}}
+{"class":"org.apache.spark.ml.evaluation.BinaryClassificationEvaluator","timestamp":1680102280456,"sparkVersion":"3.1.2","uid":"binEval_2b0597e0e4cb","paramMap":{"labelCol":"z_isMatch"},"defaultParamMap":{"rawPredictionCol":"rawPrediction","labelCol":"label","numBins":1000,"metricName":"areaUnderROC"}}

models/103/model/classifier/best.model/metadata/part-00000

@@ -1 +1 @@
-{"class":"org.apache.spark.ml.tuning.CrossValidatorModel","timestamp":1658797718446,"sparkVersion":"3.1.2","uid":"cv_21102e46a666","paramMap":{"numFolds":2,"seed":-1191137437,"foldCol":"","estimatorParamMaps":[[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"1.0E-4","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.4","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"1.0E-4","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.45","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"1.0E-4","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.5","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"1.0E-4","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.55","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"0.001","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.4","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"0.001","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.45","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"0.001","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.5","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"0.001","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.55","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"0.01","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.4","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"0.01","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.45","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"0.01","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.5","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"0.01","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.55","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"0.1","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.4","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"0.1","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.45","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"0.1","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.5","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"0.1","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.55","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"1.0","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.4","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"1.0","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.45","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"1.0","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.5","isJson":"true"}],[{"parent":"logreg_a2f6b7295985","name":"regParam","value":"1.0","isJson":"true"},{"parent":"logreg_a2f6b7295985","name":"threshold","value":"0.55","isJson":"true"}]]},"defaultParamMap":{"numFolds":3,"seed":-1191137437,"foldCol":""},"avgMetrics":[0.6234817813765183,0.6234817813765183,0.6234817813765183,0.6234817813765183,0.6060222672064778,0.6060222672064778,0.6060222672064778,0.6060222672064778,0.6629554655870447,0.6629554655870447,0.6629554655870447,0.6629554655870447,0.7687246963562754,0.7687246963562754,0.7687246963562754,0.7687246963562754,0.7820091093117408,0.7820091093117408,0.7820091093117408,0.7820091093117408],"persistSubModels":false}
+{"class":"org.apache.spark.ml.tuning.CrossValidatorModel","timestamp":1680102280126,"sparkVersion":"3.1.2","uid":"cv_91c0714c7144","paramMap":{"foldCol":"","seed":-1191137437,"numFolds":2,"estimatorParamMaps":[[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.4","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"1.0E-4","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.45","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"1.0E-4","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.5","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"1.0E-4","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.55","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"1.0E-4","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.4","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"0.001","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.45","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"0.001","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.5","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"0.001","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.55","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"0.001","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.4","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"0.01","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.45","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"0.01","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.5","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"0.01","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.55","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"0.01","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.4","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"0.1","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.45","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"0.1","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.5","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"0.1","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.55","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"0.1","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.4","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"1.0","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.45","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"1.0","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.5","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"1.0","isJson":"true"}],[{"parent":"logreg_3ad85ac653f2","name":"threshold","value":"0.55","isJson":"true"},{"parent":"logreg_3ad85ac653f2","name":"regParam","value":"1.0","isJson":"true"}]]},"defaultParamMap":{"foldCol":"","seed":-1191137437,"numFolds":3},"avgMetrics":[0.5819838056680162,0.5819838056680162,0.5819838056680162,0.5819838056680162,0.6077935222672064,0.6077935222672064,0.6077935222672064,0.6077935222672064,0.6620698380566802,0.6620698380566802,0.6620698380566802,0.6620698380566802,0.7555668016194332,0.7555668016194332,0.7555668016194332,0.7555668016194332,0.768092105263158,0.768092105263158,0.768092105263158,0.768092105263158],"persistSubModels":false}