renamed package to more suited and camelcase name

SANSA-Stack · Aug 31, 2020 · b8bdeea · b8bdeea
1 parent 21e59a5
commit b8bdeea
Show file tree

Hide file tree

Showing 13 changed files with 89 additions and 12 deletions.
diff --git a/...a-ml-spark/src/main/scala/net/sansa_stack/ml/spark/similarity/examples/minimalCalls.scala b/...a-ml-spark/src/main/scala/net/sansa_stack/ml/spark/similarity/examples/minimalCalls.scala
@@ -1,6 +1,6 @@
 package net.sansa_stack.ml.spark.similarity.examples
 
-import net.sansa_stack.ml.spark.similarity.similarity_measures.{BatetModel, BraunBlanquetModel, DiceModel, JaccardModel, OchiaiModel, SimpsonModel, TverskyModel}
+import net.sansa_stack.ml.spark.similarity.similarityEstimationModels.{BatetModel, BraunBlanquetModel, DiceModel, JaccardModel, OchiaiModel, SimpsonModel, TverskyModel}
 import net.sansa_stack.ml.spark.utils.FeatureExtractorModel
 import org.apache.jena.riot.Lang
 import org.apache.spark.sql.{DataFrame, Row, SparkSession}

diff --git a/...n/scala/net/sansa_stack/ml/spark/similarity/experiment/SimilarityPipelineExperiment.scala b/...n/scala/net/sansa_stack/ml/spark/similarity/experiment/SimilarityPipelineExperiment.scala
@@ -2,7 +2,7 @@ package net.sansa_stack.ml.spark.similarity.experiment
 
 import java.util.Calendar
 
-import net.sansa_stack.ml.spark.similarity.similarity_measures.{JaccardModel, TverskyModel}
+import net.sansa_stack.ml.spark.similarity.similarityEstimationModels.{JaccardModel, TverskyModel}
 import net.sansa_stack.ml.spark.utils.{ConfigResolver, FeatureExtractorModel, FileLister}
 import net.sansa_stack.rdf.spark.io._
 import org.apache.jena.riot.Lang

diff --git a/sansa-ml-spark/src/main/scala/net/sansa_stack/ml/spark/similarity/run/Jaccard.scala b/sansa-ml-spark/src/main/scala/net/sansa_stack/ml/spark/similarity/run/Jaccard.scala
@@ -4,7 +4,7 @@ import java.io.File
 import java.util.Calendar
 
 import com.typesafe.config.ConfigFactory
-import net.sansa_stack.ml.spark.similarity.similarity_measures.JaccardModel
+import net.sansa_stack.ml.spark.similarity.similarityEstimationModels.JaccardModel
 import net.sansa_stack.ml.spark.utils.{FeatureExtractorModel, SimilarityExperimentMetaGraphFactory}
 import net.sansa_stack.rdf.spark.io._
 import org.apache.jena.riot.Lang

diff --git a/sansa-ml-spark/src/main/scala/net/sansa_stack/ml/spark/similarity/run/Resnik.scala b/sansa-ml-spark/src/main/scala/net/sansa_stack/ml/spark/similarity/run/Resnik.scala
@@ -0,0 +1,77 @@
+package net.sansa_stack.ml.spark.similarity.run
+
+import net.sansa_stack.ml.spark.utils.FeatureExtractorModel
+import net.sansa_stack.owl.spark.dataset
+import net.sansa_stack.rdf.spark.io._
+import org.apache.jena.riot.Lang
+import org.apache.spark.ml.feature.{CountVectorizer, CountVectorizerModel, IDF}
+import org.apache.spark.ml.linalg.Vector
+import org.apache.spark.sql.functions.{col, udf}
+import org.apache.spark.sql.types.DataTypes
+import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
+
+object Resnik {
+  def main(args: Array[String]): Unit = {
+
+    // start spark session
+    val spark = SparkSession.builder
+      .appName(s"JaccardSimilarityEvaluation")
+      .master("local[*]")
+      .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
+      .getOrCreate()
+
+    // define inputpath if it is not parameter
+    val inputPath = "/Users/carstendraschner/GitHub/SANSA-ML/sansa-ml-spark/src/main/resources/movie.nt"
+
+    // read in data as Data`Frame
+    val triplesDf: DataFrame = spark.read.rdf(Lang.NTRIPLES)(inputPath)
+
+    triplesDf.show()
+
+    // feature extraction
+    val featureExtractorModel = new FeatureExtractorModel()
+      .setMode("an")
+    val extractedFeaturesDataFrame = featureExtractorModel.transform(triplesDf)
+    extractedFeaturesDataFrame.show()
+
+    // count Vectorization
+    val cvModel: CountVectorizerModel = new CountVectorizer()
+      .setInputCol("extractedFeatures")
+      .setOutputCol("vectorizedFeatures")
+      .fit(extractedFeaturesDataFrame)
+    val tmpCvDf: DataFrame = cvModel.transform(extractedFeaturesDataFrame)
+    val isNoneZeroVector = udf({ v: Vector => v.numNonzeros > 0 }, DataTypes.BooleanType)
+    val countVectorizedFeaturesDataFrame: DataFrame = tmpCvDf.filter(isNoneZeroVector(col("vectorizedFeatures"))).select("uri", "vectorizedFeatures")
+    countVectorizedFeaturesDataFrame.show()
+
+    // similarity estimations
+    // for nearestNeighbors we need one key which is a Vector to search for NN
+    val sample_key: Vector = countVectorizedFeaturesDataFrame.take(1)(0).getAs[Vector]("vectorizedFeatures")
+
+    val idf = new IDF().setInputCol("vectorizedFeatures").setOutputCol("features")
+    val idfModel = idf.fit(countVectorizedFeaturesDataFrame)
+
+    val rescaledData = idfModel.transform(countVectorizedFeaturesDataFrame)
+    rescaledData.show(false)
+
+    spark.stop()
+
+  }
+}
+/*
+class InformationContentModel {
+  def calcIC(df: DataFrame): DataFrame = {
+
+    val ds: Dataset[(String, String, String)] = dataset.as[(String, String, String)]
+    // collect all element occurences
+    val drdd = ds.rdd
+
+    val occurences = drdd.map(_._3).filter(!_.contains("\""))
+
+    val occurenceMap = drdd
+      .flatMap(t => Seq((t._1, 1), (t._3, 1)))
+
+    val numberOccurences = occurenceMap.reduceByKey(_ + _)
+
+  }
+}*/
diff --git a/sansa-ml-spark/src/main/scala/net/sansa_stack/ml/spark/similarity/run/Tversky.scala b/sansa-ml-spark/src/main/scala/net/sansa_stack/ml/spark/similarity/run/Tversky.scala
@@ -2,7 +2,7 @@ package net.sansa_stack.ml.spark.similarity.run
 
 import java.util.Calendar
 
-import net.sansa_stack.ml.spark.similarity.similarity_measures.TverskyModel
+import net.sansa_stack.ml.spark.similarity.similarityEstimationModels.TverskyModel
 import net.sansa_stack.ml.spark.utils.{FeatureExtractorModel, SimilarityExperimentMetaGraphFactory}
 import net.sansa_stack.rdf.spark.io._
 import org.apache.jena.riot.Lang

diff --git a/...rity/similarity_measures/BatetModel.scala → ...milarityEstimationModels/BatetModel.scala b/...rity/similarity_measures/BatetModel.scala → ...milarityEstimationModels/BatetModel.scala
@@ -1,4 +1,4 @@
-package net.sansa_stack.ml.spark.similarity.similarity_measures
+package net.sansa_stack.ml.spark.similarity.similarityEstimationModels
 
 import org.apache.spark.ml.linalg.Vector
 import org.apache.spark.sql.DataFrame

diff --git a/...ilarity_measures/BraunBlanquetModel.scala → ...EstimationModels/BraunBlanquetModel.scala b/...ilarity_measures/BraunBlanquetModel.scala → ...EstimationModels/BraunBlanquetModel.scala
@@ -1,4 +1,4 @@
-package net.sansa_stack.ml.spark.similarity.similarity_measures
+package net.sansa_stack.ml.spark.similarity.similarityEstimationModels
 
 import org.apache.spark.ml.linalg.Vector
 import org.apache.spark.sql.DataFrame

diff --git a/...arity/similarity_measures/DiceModel.scala → ...imilarityEstimationModels/DiceModel.scala b/...arity/similarity_measures/DiceModel.scala → ...imilarityEstimationModels/DiceModel.scala
@@ -1,4 +1,4 @@
-package net.sansa_stack.ml.spark.similarity.similarity_measures
+package net.sansa_stack.ml.spark.similarity.similarityEstimationModels
 
 import org.apache.spark.ml.linalg.Vector
 import org.apache.spark.sql.DataFrame

diff --git a/...res/GenericSimilarityEstimatorModel.scala → ...els/GenericSimilarityEstimatorModel.scala b/...res/GenericSimilarityEstimatorModel.scala → ...els/GenericSimilarityEstimatorModel.scala
@@ -1,4 +1,4 @@
-package net.sansa_stack.ml.spark.similarity.similarity_measures
+package net.sansa_stack.ml.spark.similarity.similarityEstimationModels
 
 import org.apache.spark.ml.linalg.Vector
 import org.apache.spark.sql.functions.{col, udf, lit, typedLit}

diff --git a/...ty/similarity_measures/JaccardModel.scala → ...larityEstimationModels/JaccardModel.scala b/...ty/similarity_measures/JaccardModel.scala → ...larityEstimationModels/JaccardModel.scala
@@ -1,4 +1,4 @@
-package net.sansa_stack.ml.spark.similarity.similarity_measures
+package net.sansa_stack.ml.spark.similarity.similarityEstimationModels
 
 import org.apache.spark
 import org.apache.spark.ml.linalg.Vector

diff --git a/...ity/similarity_measures/OchiaiModel.scala → ...ilarityEstimationModels/OchiaiModel.scala b/...ity/similarity_measures/OchiaiModel.scala → ...ilarityEstimationModels/OchiaiModel.scala
@@ -1,4 +1,4 @@
-package net.sansa_stack.ml.spark.similarity.similarity_measures
+package net.sansa_stack.ml.spark.similarity.similarityEstimationModels
 
 import org.apache.spark.ml.linalg.Vector
 import org.apache.spark.sql.DataFrame

diff --git a/...ty/similarity_measures/SimpsonModel.scala → ...larityEstimationModels/SimpsonModel.scala b/...ty/similarity_measures/SimpsonModel.scala → ...larityEstimationModels/SimpsonModel.scala
@@ -1,4 +1,4 @@
-package net.sansa_stack.ml.spark.similarity.similarity_measures
+package net.sansa_stack.ml.spark.similarity.similarityEstimationModels
 
 import org.apache.spark.ml.linalg.Vector
 import org.apache.spark.sql.DataFrame

diff --git a/...ty/similarity_measures/TverskyModel.scala → ...larityEstimationModels/TverskyModel.scala b/...ty/similarity_measures/TverskyModel.scala → ...larityEstimationModels/TverskyModel.scala
@@ -1,4 +1,4 @@
-package net.sansa_stack.ml.spark.similarity.similarity_measures
+package net.sansa_stack.ml.spark.similarity.similarityEstimationModels
 
 import org.apache.spark.ml.linalg.Vector
 import org.apache.spark.sql.DataFrame