From e66753c3ac016dbd855975fd5ef66b9c464c16a6 Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Wed, 17 Aug 2016 17:37:37 +0900 Subject: [PATCH 1/3] remove mysql-connector-java from dependencies. fix minor bug when arise after change default database from mysql to h2. --- s2core/build.sbt | 1 - s2core/src/main/resources/reference.conf | 5 +- .../scala/org/apache/s2graph/core/Graph.scala | 4 +- .../s2graph/core/mysqls/ExperimentSpec.scala | 6 +- s2counter_core/build.sbt | 2 +- s2rest_netty/conf/reference.conf | 2 +- .../src/main/resources/reference.conf | 2 +- s2rest_play/conf/reference.conf | 2 +- .../rest/play/benchmark/BenchmarkCommon.scala | 5 + .../rest/play/benchmark/GraphUtilSpec.scala | 110 +++++++++--------- .../benchmark/SamplingBenchmarkSpec.scala | 8 +- 11 files changed, 75 insertions(+), 72 deletions(-) diff --git a/s2core/build.sbt b/s2core/build.sbt index 46c27955..02e5a0ec 100644 --- a/s2core/build.sbt +++ b/s2core/build.sbt @@ -36,7 +36,6 @@ libraryDependencies ++= Seq( "commons-pool" % "commons-pool" % "1.6", "org.scalatest" %% "scalatest" % "2.2.4" % "test", "org.scalikejdbc" %% "scalikejdbc" % "2.1.+", - "mysql" % "mysql-connector-java" % "5.1.28", "com.h2database" % "h2" % "1.4.192", "com.github.danielwegener" % "logback-kafka-appender" % "0.0.4" ) diff --git a/s2core/src/main/resources/reference.conf b/s2core/src/main/resources/reference.conf index 1840306a..86dfa672 100644 --- a/s2core/src/main/resources/reference.conf +++ b/s2core/src/main/resources/reference.conf @@ -46,8 +46,9 @@ cache.max.size=100000 # DB s2graph.models.table.name = "models-dev" -db.default.driver = "com.mysql.jdbc.Driver" -db.default.url = "jdbc:mysql://"${host}":3306/graph_dev" + +db.default.driver = "org.h2.Driver" +db.default.url="jdbc:h2:file:./var/metastore;MODE=MYSQL" db.default.user = "graph" db.default.password = "graph" diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Graph.scala b/s2core/src/main/scala/org/apache/s2graph/core/Graph.scala index f4ce7b44..aee0e956 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/Graph.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/Graph.scala @@ -43,8 +43,8 @@ object Graph { "hbase.table.name" -> "s2graph", "hbase.table.compression.algorithm" -> "gz", "phase" -> "dev", - "db.default.driver" -> "com.mysql.jdbc.Driver", - "db.default.url" -> "jdbc:mysql://localhost:3306/graph_dev", + "db.default.driver" -> "org.h2.Driver", + "db.default.url" -> "jdbc:h2:file:./var/metastore;MODE=MYSQL", "db.default.password" -> "graph", "db.default.user" -> "graph", "cache.max.size" -> java.lang.Integer.valueOf(10000), diff --git a/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala b/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala index 4a46e0ed..229d9bd3 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/mysqls/ExperimentSpec.scala @@ -40,10 +40,10 @@ class ExperimentSpec extends FlatSpec with Matchers with BeforeAndAfterAll { sql"""DELETE FROM buckets""".update().apply() sql"""DELETE FROM experiments""".update().apply() - val expId = sql"""INSERT INTO experiments(service_id, service_name, `name`, description) VALUES(1, "s1", "exp1", "")""".updateAndReturnGeneratedKey().apply() + val expId = sql"""INSERT INTO experiments(service_id, service_name, name, description) VALUES(1, 's1', 'exp1', '')""".updateAndReturnGeneratedKey().apply() sql"""INSERT INTO buckets(experiment_id, modular, http_verb, api_path, request_body, impression_id) - VALUES($expId, "1~100", "POST", "/a/b/c", "None", "imp1")""".update().apply() + VALUES($expId, '1~100', 'POST', '/a/b/c', 'None', 'imp1')""".update().apply() } @@ -63,7 +63,7 @@ class ExperimentSpec extends FlatSpec with Matchers with BeforeAndAfterAll { implicit val session = AutoSession - sql"""UPDATE buckets SET impression_id = "imp2" WHERE id = ${bucket.id}""".update().apply() + sql"""UPDATE buckets SET impression_id = 'imp2' WHERE id = ${bucket.id}""".update().apply() } // sleep ttl time diff --git a/s2counter_core/build.sbt b/s2counter_core/build.sbt index d4c0bdf7..9a75faf9 100644 --- a/s2counter_core/build.sbt +++ b/s2counter_core/build.sbt @@ -38,7 +38,7 @@ libraryDependencies ++= Seq( "org.apache.hadoop" % "hadoop-hdfs" % Common.hadoopVersion, "redis.clients" % "jedis" % "2.6.0", "org.apache.kafka" % "kafka-clients" % "0.8.2.0", - "mysql" % "mysql-connector-java" % "5.1.28", + "com.h2database" % "h2" % "1.4.192", "org.scalikejdbc" %% "scalikejdbc" % "2.1.+", "org.specs2" %% "specs2-core" % "3.6" % "test", "org.scalatest" %% "scalatest" % "2.2.1" % "test" diff --git a/s2rest_netty/conf/reference.conf b/s2rest_netty/conf/reference.conf index 9b37eda4..4a5080fb 100644 --- a/s2rest_netty/conf/reference.conf +++ b/s2rest_netty/conf/reference.conf @@ -86,7 +86,7 @@ host=localhost # DB s2graph.models.table.name="models-dev" hbase.zookeeper.quorum=${host} -db.default.url="jdbc:mysql://"${host}":3306/graph_dev" + # Query server is.query.server=true is.write.server=true diff --git a/s2rest_netty/src/main/resources/reference.conf b/s2rest_netty/src/main/resources/reference.conf index 9b37eda4..4a5080fb 100644 --- a/s2rest_netty/src/main/resources/reference.conf +++ b/s2rest_netty/src/main/resources/reference.conf @@ -86,7 +86,7 @@ host=localhost # DB s2graph.models.table.name="models-dev" hbase.zookeeper.quorum=${host} -db.default.url="jdbc:mysql://"${host}":3306/graph_dev" + # Query server is.query.server=true is.write.server=true diff --git a/s2rest_play/conf/reference.conf b/s2rest_play/conf/reference.conf index 0a54fb5a..bda503c7 100644 --- a/s2rest_play/conf/reference.conf +++ b/s2rest_play/conf/reference.conf @@ -86,7 +86,7 @@ host=localhost # DB s2graph.models.table.name="models-dev" hbase.zookeeper.quorum=${host} -db.default.url="jdbc:mysql://"${host}":3306/graph_dev" + # Query server is.query.server=true is.write.server=true diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/BenchmarkCommon.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/BenchmarkCommon.scala index 4662e48b..240421c3 100644 --- a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/BenchmarkCommon.scala +++ b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/BenchmarkCommon.scala @@ -19,7 +19,12 @@ package org.apache.s2graph.rest.play.benchmark +import com.typesafe.config.{ConfigFactory, Config} +import org.apache.s2graph.core.{Management, Graph} import org.specs2.mutable.Specification +import scalikejdbc.AutoSession + +import scala.concurrent.ExecutionContext trait BenchmarkCommon extends Specification { val wrapStr = s"\n==================================================" diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala index 75026fa4..737b8284 100644 --- a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala +++ b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/GraphUtilSpec.scala @@ -22,7 +22,7 @@ package org.apache.s2graph.rest.play.benchmark import org.apache.hadoop.hbase.util.Bytes import org.apache.s2graph.core.GraphUtil import org.apache.s2graph.core.types.{HBaseType, InnerVal, SourceVertexId} -import play.api.test.{FakeApplication, PlaySpecification} +import play.api.test.PlaySpecification import scala.collection.mutable import scala.collection.mutable.ListBuffer @@ -65,68 +65,68 @@ class GraphUtilSpec extends BenchmarkCommon with PlaySpecification { } "test murmur hash skew2" in { - running(FakeApplication()) { - import HBaseType._ - val testNum = 1000000L - val regionCount = 40 - val window = Int.MaxValue / regionCount - val rangeBytes = new ListBuffer[(List[Byte], List[Byte])]() - for { - i <- (0 until regionCount) - } yield { - val startKey = Bytes.toBytes(i * window) - val endKey = Bytes.toBytes((i + 1) * window) - rangeBytes += (startKey.toList -> endKey.toList) - } + + import HBaseType._ + val testNum = 1000000L + val regionCount = 40 + val window = Int.MaxValue / regionCount + val rangeBytes = new ListBuffer[(List[Byte], List[Byte])]() + for { + i <- (0 until regionCount) + } yield { + val startKey = Bytes.toBytes(i * window) + val endKey = Bytes.toBytes((i + 1) * window) + rangeBytes += (startKey.toList -> endKey.toList) + } - val stats = new collection.mutable.HashMap[Int, ((List[Byte], List[Byte]), Long)]() - val counts = new collection.mutable.HashMap[Short, Long]() - stats += (0 -> (rangeBytes.head -> 0L)) - - for (i <- (0L until testNum)) { - val vertexId = SourceVertexId(DEFAULT_COL_ID, InnerVal.withLong(i, HBaseType.DEFAULT_VERSION)) - val bytes = vertexId.bytes - val shortKey = GraphUtil.murmur3(vertexId.innerId.toIdString()) - val shortVal = counts.getOrElse(shortKey, 0L) + 1L - counts += (shortKey -> shortVal) - var j = 0 - var found = false - while (j < rangeBytes.size && !found) { - val (start, end) = rangeBytes(j) - if (between(bytes, start.toArray, end.toArray)) { - found = true - } - j += 1 - } - val head = rangeBytes(j - 1) - val key = j - 1 - val value = stats.get(key) match { - case None => 0L - case Some(v) => v._2 + 1 + val stats = new collection.mutable.HashMap[Int, ((List[Byte], List[Byte]), Long)]() + val counts = new collection.mutable.HashMap[Short, Long]() + stats += (0 -> (rangeBytes.head -> 0L)) + + for (i <- (0L until testNum)) { + val vertexId = SourceVertexId(DEFAULT_COL_ID, InnerVal.withLong(i, HBaseType.DEFAULT_VERSION)) + val bytes = vertexId.bytes + val shortKey = GraphUtil.murmur3(vertexId.innerId.toIdString()) + val shortVal = counts.getOrElse(shortKey, 0L) + 1L + counts += (shortKey -> shortVal) + var j = 0 + var found = false + while (j < rangeBytes.size && !found) { + val (start, end) = rangeBytes(j) + if (between(bytes, start.toArray, end.toArray)) { + found = true } - stats += (key -> (head, value)) + j += 1 } - val sorted = stats.toList.sortBy(kv => kv._2._2).reverse - println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount") - sorted.foreach { case (idx, ((start, end), cnt)) => - val startShort = Bytes.toShort(start.take(2).toArray) - val endShort = Bytes.toShort(end.take(2).toArray) - val count = counts.count(t => startShort <= t._1 && t._1 < endShort) - println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count") - + val head = rangeBytes(j - 1) + val key = j - 1 + val value = stats.get(key) match { + case None => 0L + case Some(v) => v._2 + 1 } - println("\n" * 10) - println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount") - stats.toList.sortBy(kv => kv._1).reverse.foreach { case (idx, ((start, end), cnt)) => - val startShort = Bytes.toShort(start.take(2).toArray) - val endShort = Bytes.toShort(end.take(2).toArray) - val count = counts.count(t => startShort <= t._1 && t._1 < endShort) - println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count") + stats += (key ->(head, value)) + } + val sorted = stats.toList.sortBy(kv => kv._2._2).reverse + println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount") + sorted.foreach { case (idx, ((start, end), cnt)) => + val startShort = Bytes.toShort(start.take(2).toArray) + val endShort = Bytes.toShort(end.take(2).toArray) + val count = counts.count(t => startShort <= t._1 && t._1 < endShort) + println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count") - } } + println("\n" * 10) + println(s"Index: StartBytes ~ EndBytes\tStartShortBytes ~ EndShortBytes\tStartShort ~ EndShort\tCount\tShortCount") + stats.toList.sortBy(kv => kv._1).reverse.foreach { case (idx, ((start, end), cnt)) => + val startShort = Bytes.toShort(start.take(2).toArray) + val endShort = Bytes.toShort(end.take(2).toArray) + val count = counts.count(t => startShort <= t._1 && t._1 < endShort) + println(s"$idx: $start ~ $end\t${start.take(2)} ~ ${end.take(2)}\t$startShort ~ $endShort\t$cnt\t$count") + + } + true } diff --git a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/SamplingBenchmarkSpec.scala b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/SamplingBenchmarkSpec.scala index 1cda2778..b5d6691d 100644 --- a/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/SamplingBenchmarkSpec.scala +++ b/s2rest_play/test/org/apache/s2graph/rest/play/benchmark/SamplingBenchmarkSpec.scala @@ -26,9 +26,8 @@ import scala.util.Random class SamplingBenchmarkSpec extends BenchmarkCommon with PlaySpecification { "sample" should { - implicit val app = FakeApplication() - "sample benchmark" in new WithApplication(app) { + "sample benchmark" in { @tailrec def randomInt(n: Int, range: Int, set: Set[Int] = Set.empty[Int]): Set[Int] = { if (set.size == n) set @@ -61,7 +60,7 @@ class SamplingBenchmarkSpec extends BenchmarkCommon with PlaySpecification { while (m < num) { val u = Random.nextDouble() - if ( (N - t)*u < num - m) { + if ((N - t) * u < num - m) { sampled = ls(t) :: sampled m += 1 } @@ -99,8 +98,7 @@ class SamplingBenchmarkSpec extends BenchmarkCommon with PlaySpecification { val sampled = rngSample(testNum, testData) } } + true } - - } } From fcd9fb97b295f723cfcddc50199d66797ef6146f Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Tue, 23 Aug 2016 10:31:04 +0900 Subject: [PATCH 2/3] change -1 limit on QueryTest. --- .../scala/org/apache/s2graph/core/Integrate/QueryTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala index 8bdc99b4..c0071fa0 100644 --- a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/QueryTest.scala @@ -307,7 +307,7 @@ class QueryTest extends IntegrateCommon with BeforeAndAfterEach { "label": "$testLabelName", "direction": "in", "offset": 0, - "limit": -1 + "limit": 10 } ]] }""".stripMargin) From b68b141bca05cfaebf09b397cc3e47961bad2a4c Mon Sep 17 00:00:00 2001 From: DO YUNG YOON Date: Mon, 29 Aug 2016 03:47:36 +0900 Subject: [PATCH 3/3] update CHANGES. --- CHANGES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES b/CHANGES index e68a0e58..3aab524f 100644 --- a/CHANGES +++ b/CHANGES @@ -149,6 +149,8 @@ Release 0.12.1 - unreleased S2GRAPH-89: Change version number on project. (Committed by DOYUNG YOON). S2GRAPH-91: Remove custom repository on project's build.sbt (Committed by DOYUNG YOON). + + S2GRAPH-93: Remove MySQL JDBC driver (Committed by DOYUNG YOON). TEST