Merge f2062b3 into 4851b2e

build.sbt

@@ -2,7 +2,7 @@ lazy val scala211 = "2.11.8"
 lazy val supportedScalaVersions = List(scala211)
 lazy val sparkVersion = "2.3.1"
 
-ThisBuild / organization := "org.noam.shaish"
+ThisBuild / organization := "org.nevair"
 ThisBuild / version := "0.1.0-SNAPSHOT"
 
 

core/src/main/scala/org/noam/shaish/containers/Container.scala → core/src/main/scala/org/nevair/containers/Container.scala

@@ -1,10 +1,9 @@
-package org.noam.shaish.containers
+package org.nevair.containers
 
 import scala.annotation.implicitNotFound
 import scala.language.higherKinds
 import scala.reflect.ClassTag
 
-
 /**
   * Created by Noam Shaish on 6/25/2017.
   */
@@ -75,34 +74,4 @@ object Container {
   implicit val listContainer: Container[List] = new ListContainer
 
   implicit val listContainerToPairListContainer: PairContainer[List] = new PairListContainer
-
-  implicit object SetContainer extends Container[Set] {
-    override def map[A, B: ClassTag](c: Set[A])(f: (A) => B): Set[B] = c.map(f)
-
-    override def repartition[A](c: Set[A])(repNumber: Int): Set[A] = c
-
-    override def flatMap[A, B: ClassTag](c: Set[A])(f: (A) => TraversableOnce[B]): Set[B] = c.flatMap(f)
-
-    override def filter[A](c: Set[A])(p: (A) => Boolean): Set[A] = c.filter(p)
-
-    override def count[A](c: Set[A])(p: (A) => Boolean = (_: A) => true): Long = c.count(p).toLong
-
-    override def reduce[A](c: Set[A])(f: (A, A) => A): A = c.reduce(f)
-  }
-
-  implicit object VectorContainer extends Container[Vector] {
-    override def map[A, B: ClassTag](c: Vector[A])(f: (A) => B): Vector[B] = c.map(f)
-
-    override def repartition[A](c: Vector[A])(repNumber: Int): Vector[A] = c
-
-    override def flatMap[A, B: ClassTag](c: Vector[A])(f: (A) => TraversableOnce[B]): Vector[B] = c.flatMap(f)
-
-    override def filter[A](c: Vector[A])(f: (A) => Boolean): Vector[A] = c.filter(f)
-
-    override def count[A](c: Vector[A])(p: (A) => Boolean = (_: A) => true): Long = c.count(p).toLong
-
-    override def reduce[A](c: Vector[A])(f: (A, A) => A): A = c.reduce(f)
-  }
-
-
 }

core/src/test/scala/org/nevair/containers/ContainerTest.scala

@@ -0,0 +1,68 @@
+package org.nevair.containers
+
+import org.scalatest.{FlatSpec, Matchers}
+import Container.ops._
+
+import scala.language.higherKinds
+
+class ContainerTest extends FlatSpec with Matchers {
+  behavior of "List Container"
+
+  it should "call map on list" in {
+    def mapIt[C[_]: Container](lContainer: C[Int])(f: Int => String): C[String] = lContainer.map(f)
+
+    val l = List(1, 2, 3, 4)
+
+    mapIt(l)(_.toString) should contain theSameElementsInOrderAs l.map(_.toString)
+  }
+
+  it should "call flatMap on list" in {
+    def flatMapIt[C[_]: Container](lContainer: C[(Int, Int)]): C[Int] = lContainer.flatMap(t => Seq(t._1, t._2))
+
+    val l = List((1, 2), (3, 4))
+
+    flatMapIt(l) should contain theSameElementsInOrderAs l.flatMap(t => Seq(t._1, t._2))
+  }
+
+  it should "do nothing when calling repartition on list" in {
+    def repartitionIt[C[_]: Container](lContainer: C[Int]): C[Int] = lContainer.repartition(1000)
+
+    val l = List(1, 2, 3, 4)
+
+    repartitionIt(l) should contain theSameElementsInOrderAs l
+  }
+
+  it should "count all elements without filter" in {
+    def countItAll[C[_]: Container](lContainer: C[Int]): Long = lContainer.count(_ => true)
+
+    val l = List(1, 2, 3, 4)
+
+    countItAll(l) should be(l.size)
+  }
+
+  it should "count elements fulfilling filter" in {
+    def countIt[C[_]: Container](lContainer: C[Int])(p: Int => Boolean): Long = lContainer.count(p)
+
+    val l = List(1, 2, 3, 4)
+
+    countIt(l)(_ % 2 == 0) should be(l.count(_ % 2 == 0))
+  }
+
+  it should "call reduce on list" in {
+    def reduceIt[C[_]: Container](lContainer: C[Int])(f: (Int, Int) => Int): Int = lContainer.reduce(f)
+
+    val l = List(1, 2, 3, 4)
+
+    reduceIt(l)(_ + _) should be(l.sum)
+  }
+
+  behavior of "List PairContainer"
+
+  it should "reduce by key properly" in {
+    def sumIt[C[_]: PairContainer](lContainer: C[(String, Int)]): C[(String, Int)] = lContainer.reduceByKey(_ + _)
+
+    val l = List(("a", 1), ("b", 1), ("a", 1), ("c", 1))
+
+    sumIt(l) should contain theSameElementsAs List(("a", 2), ("b", 1), ("c", 1))
+  }
+}

example/src/main/scala/org/nevair/containers/example/ExampleContainer.scala

@@ -0,0 +1,35 @@
+package org.nevair.containers.example
+
+import org.nevair.containers.Container
+
+import scala.reflect.ClassTag
+
+object ExampleContainer {
+  implicit object SetContainer extends Container[Set] {
+    override def map[A, B: ClassTag](c: Set[A])(f: (A) => B): Set[B] = c.map(f)
+
+    override def repartition[A](c: Set[A])(repNumber: Int): Set[A] = c
+
+    override def flatMap[A, B: ClassTag](c: Set[A])(f: (A) => TraversableOnce[B]): Set[B] = c.flatMap(f)
+
+    override def filter[A](c: Set[A])(p: (A) => Boolean): Set[A] = c.filter(p)
+
+    override def count[A](c: Set[A])(p: (A) => Boolean = (_: A) => true): Long = c.count(p).toLong
+
+    override def reduce[A](c: Set[A])(f: (A, A) => A): A = c.reduce(f)
+  }
+
+  implicit object VectorContainer extends Container[Vector] {
+    override def map[A, B: ClassTag](c: Vector[A])(f: (A) => B): Vector[B] = c.map(f)
+
+    override def repartition[A](c: Vector[A])(repNumber: Int): Vector[A] = c
+
+    override def flatMap[A, B: ClassTag](c: Vector[A])(f: (A) => TraversableOnce[B]): Vector[B] = c.flatMap(f)
+
+    override def filter[A](c: Vector[A])(f: (A) => Boolean): Vector[A] = c.filter(f)
+
+    override def count[A](c: Vector[A])(p: (A) => Boolean = (_: A) => true): Long = c.count(p).toLong
+
+    override def reduce[A](c: Vector[A])(f: (A, A) => A): A = c.reduce(f)
+  }
+}

example/src/main/scala/org/noam/shaish/containers/example/Logic.scala → example/src/main/scala/org/nevair/containers/example/Logic.scala

@@ -1,10 +1,10 @@
-package org.noam.shaish.containers.example
+package org.nevair.containers.example
 
 import java.util.Random
 
 import breeze.linalg.{DenseVector, Vector}
-import org.noam.shaish.containers.{Container, PairContainer}
-import org.noam.shaish.containers.Container.ops._
+import org.nevair.containers.{Container, PairContainer}
+import org.nevair.containers.Container.ops._
 
 import scala.language.higherKinds
 

example/src/test/scala/org/noam/shaish/containers/example/LogicTest.scala → example/src/test/scala/org/nevair/containers/example/LogicTest.scala

@@ -1,8 +1,9 @@
-package org.noam.shaish.containers.example
+package org.nevair.containers.example
 
 import org.apache.spark.{SparkConf, SparkContext}
 import org.scalatest.{FlatSpec, Matchers}
-import org.noam.shaish.containers.spark.RDDContainer._
+import org.nevair.containers.spark.SparkContainer._
+import ExampleContainer._
 
 /**
   * Created by Noam Shaish on 6/25/2017.

spark/src/main/scala/org/noam/shaish/containers/spark/RDDContainer.scala → spark/src/main/scala/org/nevair/containers/spark/SparkContainer.scala

@@ -1,11 +1,11 @@
-package org.noam.shaish.containers.spark
+package org.nevair.containers.spark
 
 import org.apache.spark.rdd.RDD
-import org.noam.shaish.containers.{Container, PairContainer}
+import org.nevair.containers.{Container, PairContainer}
 
 import scala.reflect.ClassTag
 
-object RDDContainer {
+object SparkContainer {
   private sealed class RDDContainer extends Container[RDD] {
     override def map[A, B: ClassTag](c: RDD[A])(f: (A) => B): RDD[B] = c.map(f)