From 5980bf7fd2b1a5977c5badca2400cb4a08d042b8 Mon Sep 17 00:00:00 2001 From: daewon Date: Thu, 14 Jan 2016 18:03:20 +0900 Subject: [PATCH 01/13] S2GRAPH-7 Move PostProcess.scala to s2core from root project --- .../main/scala/com/kakao/s2graph/core/PostProcess.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala index 0a26d26a..3a2420a1 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala @@ -323,10 +323,17 @@ object PostProcess extends JSONParser { to <- innerValToJsValue(edge.tgtVertex.id.innerId, tgtColumn.columnType) } yield { val targetColumns = if (q.selectColumnsSet.isEmpty) reservedColumns else (reservedColumns & q.selectColumnsSet) + "props" +<<<<<<< HEAD:s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala val _propsMap = queryParam.label.metaPropsDefaultMapInner ++ propsToJson(edge, q, queryParam) val propsMap = if (q.selectColumnsSet.nonEmpty) _propsMap.filterKeys(q.selectColumnsSet) else _propsMap +======= + + val _propsMap = queryParam.label.metaPropsDefaultMapInner ++ propsToJson(edge, q, queryParam) + val propsMap = if (q.selectColumnsSet.nonEmpty) _propsMap.filterKeys(q.selectColumnsSet) else _propsMap + +>>>>>>> 90777fd... S2GRAPH-7 Move PostProcess.scala to s2core from root project:s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala val kvMap = targetColumns.foldLeft(Map.empty[String, JsValue]) { (map, column) => val jsValue = column match { case "cacheRemain" => JsNumber(queryParam.cacheTTLInMillis - (System.currentTimeMillis() - queryParam.timestamp)) From 4dfebc767f83d2056dcd90b1539ffcad750ee158 Mon Sep 17 00:00:00 2001 From: daewon Date: Tue, 5 Jan 2016 15:07:20 +0900 Subject: [PATCH 02/13] S2GRAPH-7 Move RequestParser from Root to s2core --- .../scala/com/kakao/s2graph/core/rest/RequestParser.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala index 98cc68b2..2a966e9f 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala @@ -6,6 +6,7 @@ import com.kakao.s2graph.core.mysqls._ import com.kakao.s2graph.core.parsers.WhereParser import com.kakao.s2graph.core.types._ import com.typesafe.config.Config + import play.api.libs.json._ import scala.util.{Failure, Success, Try} @@ -442,7 +443,11 @@ class RequestParser(config: Config) extends JSONParser { def toServiceElements(jsValue: JsValue) = { val serviceName = parse[String](jsValue, "serviceName") +<<<<<<< HEAD:s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala val cluster = (jsValue \ "cluster").asOpt[String].getOrElse(DefaultCluster) +======= + val cluster = (jsValue \ "cluster").asOpt[String].getOrElse(defaultCluster) +>>>>>>> e0b3e2f... S2GRAPH-7 Move RequestParser from Root to s2core:s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala val hTableName = (jsValue \ "hTableName").asOpt[String].getOrElse(s"${serviceName}-${DefaultPhase}") val preSplitSize = (jsValue \ "preSplitSize").asOpt[Int].getOrElse(1) val hTableTTL = (jsValue \ "hTableTTL").asOpt[Int] From 9f16c00a251c395e2b393932359edd18481a3a35 Mon Sep 17 00:00:00 2001 From: daewon Date: Thu, 14 Jan 2016 18:09:27 +0900 Subject: [PATCH 03/13] S2GRAPH-7 Make RestCaller for abstract over http layer --- .../scala/com/kakao/s2graph/core/rest/RequestParser.scala | 5 ----- 1 file changed, 5 deletions(-) diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala index 2a966e9f..78fc3752 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala @@ -443,11 +443,7 @@ class RequestParser(config: Config) extends JSONParser { def toServiceElements(jsValue: JsValue) = { val serviceName = parse[String](jsValue, "serviceName") -<<<<<<< HEAD:s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala val cluster = (jsValue \ "cluster").asOpt[String].getOrElse(DefaultCluster) -======= - val cluster = (jsValue \ "cluster").asOpt[String].getOrElse(defaultCluster) ->>>>>>> e0b3e2f... S2GRAPH-7 Move RequestParser from Root to s2core:s2core/src/main/scala/com/kakao/s2graph/core/rest/RequestParser.scala val hTableName = (jsValue \ "hTableName").asOpt[String].getOrElse(s"${serviceName}-${DefaultPhase}") val preSplitSize = (jsValue \ "preSplitSize").asOpt[Int].getOrElse(1) val hTableTTL = (jsValue \ "hTableTTL").asOpt[Int] @@ -487,7 +483,6 @@ class RequestParser(config: Config) extends JSONParser { } (src, tgt, QueryParam(LabelWithDirection(label.id.get, dir))) } - (quads, isReverted) } From 7453aebf0ddd47d93ac68e1ca027923978aef235 Mon Sep 17 00:00:00 2001 From: daewon Date: Thu, 14 Jan 2016 18:11:06 +0900 Subject: [PATCH 04/13] S2GRAPH-7 Move root project to s2rest_play --- s2rest_play/conf/routes | 1 + .../benchmark/PostProcessBenchmarkSpec.scala | 241 +++++++ .../controllers/AdminControllerSpec.scala | 27 + .../test/controllers/BasicCrudSpec.scala | 251 ++++++++ .../test/controllers/EdgeControllerSpec.scala | 22 + .../test/controllers/QueryCacheSpec.scala | 89 +++ s2rest_play/test/controllers/QuerySpec.scala | 604 ++++++++++++++++++ s2rest_play/test/controllers/SpecCommon.scala | 365 +++++++++++ .../controllers/StrongLabelDeleteSpec.scala | 345 ++++++++++ s2rest_play/test/controllers/VertexSpec.scala | 39 ++ .../controllers/WeakLabelDeleteSpec.scala | 126 ++++ 11 files changed, 2110 insertions(+) create mode 100644 s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala create mode 100644 s2rest_play/test/controllers/AdminControllerSpec.scala create mode 100644 s2rest_play/test/controllers/BasicCrudSpec.scala create mode 100644 s2rest_play/test/controllers/EdgeControllerSpec.scala create mode 100644 s2rest_play/test/controllers/QueryCacheSpec.scala create mode 100644 s2rest_play/test/controllers/QuerySpec.scala create mode 100644 s2rest_play/test/controllers/SpecCommon.scala create mode 100644 s2rest_play/test/controllers/StrongLabelDeleteSpec.scala create mode 100644 s2rest_play/test/controllers/VertexSpec.scala create mode 100644 s2rest_play/test/controllers/WeakLabelDeleteSpec.scala diff --git a/s2rest_play/conf/routes b/s2rest_play/conf/routes index df4a1eed..5282b7b0 100644 --- a/s2rest_play/conf/routes +++ b/s2rest_play/conf/routes @@ -25,6 +25,7 @@ POST /graphs/edges/updateWithWait contro POST /graphs/edges/increment controllers.EdgeController.increments() POST /graphs/edges/incrementCount controllers.EdgeController.incrementCounts() POST /graphs/edges/bulk controllers.EdgeController.mutateBulk() +POST /graphs/edges/bulkWithWait controllers.EdgeController.mutateBulkWithWait() ## Vertex POST /graphs/vertices/insert controllers.VertexController.inserts() diff --git a/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala b/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala new file mode 100644 index 00000000..22a7b5a1 --- /dev/null +++ b/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala @@ -0,0 +1,241 @@ +package benchmark + +import com.kakao.s2graph.core.mysqls.Label +import com.kakao.s2graph.core.rest.RequestParser +import com.kakao.s2graph.core.{PostProcess, Graph, Management} +import com.typesafe.config.ConfigFactory +import controllers._ +import play.api.libs.json.{JsValue, Json} +import play.api.test.{FakeApplication, FakeRequest, PlaySpecification} + +import scala.concurrent.Await +import scala.concurrent.duration._ + +/** + * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 6.. + */ +class PostProcessBenchmarkSpec extends SpecCommon with BenchmarkCommon with PlaySpecification { + sequential + + import Helper._ + + init() + + override def init() = { + running(FakeApplication()) { + println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") + Management.deleteService(testServiceName) + + // 1. createService + val result = AdminController.createServiceInner(Json.parse(createService)) + println(s">> Service created : $createService, $result") + + val labelNames = Map( + testLabelNameWeak -> testLabelNameWeakCreate + ) + + for { + (labelName, create) <- labelNames + } { + Management.deleteLabel(labelName) + Label.findByName(labelName, useCache = false) match { + case None => + AdminController.createLabelInner(Json.parse(create)) + case Some(label) => + println(s">> Label already exist: $create, $label") + } + } + + // create edges + val bulkEdges: String = (0 until 500).map { i => + edge"${System.currentTimeMillis()} insert e 0 $i $testLabelNameWeak"($(weight=i)) + }.mkString("\n") + + val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) + + println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") + } + } + + def getEdges(queryJson: JsValue): JsValue = { + val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get + contentAsJson(ret) + } + + val s2: Graph = com.kakao.s2graph.rest.Global.s2graph + +// "test performance of getEdges orderBy" >> { +// running(FakeApplication()) { +// val strJs = +// s""" +// |{ +// | "orderBy": [ +// | {"score": "DESC"}, +// | {"timestamp": "DESC"} +// | ], +// | "srcVertices": [ +// | { +// | "serviceName": "$testServiceName", +// | "columnName": "$testColumnName", +// | "ids": [0] +// | } +// | ], +// | "steps": [ +// | { +// | "step": [ +// | { +// | "cacheTTL": 60000, +// | "label": "$testLabelNameWeak", +// | "offset": 0, +// | "limit": -1, +// | "direction": "out", +// | "scoring": [ +// | {"weight": 1} +// | ] +// | } +// | ] +// | } +// | ] +// |} +// """.stripMargin +// +// object Parser extends RequestParser +// +// val js = Json.parse(strJs) +// +// val q = Parser.toQuery(js) +// +// val queryResultLs = Await.result(s2.getEdges(q), 1 seconds) +// +// val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs) +// +// (resultJs \ "size").as[Int] must_== 500 +// +// (0 to 5) foreach { _ => +// duration("toSimpleVertexArrJson new orderBy") { +// (0 to 1000) foreach { _ => +// PostProcess.toSimpleVertexArrJson(queryResultLs, Nil) +// } +// } +// } +// +// (resultJs \ "size").as[Int] must_== 500 +// } +// } + + "test performance of getEdges" >> { + running(FakeApplication()) { + val strJs = + s""" + |{ + | "srcVertices": [ + | { + | "serviceName": "$testServiceName", + | "columnName": "$testColumnName", + | "ids": [0] + | } + | ], + | "steps": [ + | { + | "step": [ + | { + | "cacheTTL": 60000, + | "label": "$testLabelNameWeak", + | "offset": 0, + | "limit": -1, + | "direction": "out", + | "scoring": [ + | {"weight": 1} + | ] + | } + | ] + | } + | ] + |} + """.stripMargin + + val config = ConfigFactory.load() + val requestParser = new RequestParser(config) + + val js = Json.parse(strJs) + + val q = requestParser.toQuery(js) + + val queryResultLs = Await.result(s2.getEdges(q), 1 seconds) + + val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs, Nil) + + (0 to 5) foreach { _ => + duration("toSimpleVertexArrJson new") { + (0 to 1000) foreach { _ => + PostProcess.toSimpleVertexArrJson(queryResultLs, Nil) + } + } + } + + (resultJs \ "size").as[Int] must_== 500 + } + } + +// "test performance of getEdges withScore=false" >> { +// running(FakeApplication()) { +// val strJs = +// s""" +// |{ +// | "withScore": false, +// | "srcVertices": [ +// | { +// | "serviceName": "$testServiceName", +// | "columnName": "$testColumnName", +// | "ids": [0] +// | } +// | ], +// | "steps": [ +// | { +// | "step": [ +// | { +// | "cacheTTL": 60000, +// | "label": "$testLabelNameWeak", +// | "offset": 0, +// | "limit": -1, +// | "direction": "out", +// | "scoring": [ +// | {"weight": 1} +// | ] +// | } +// | ] +// | } +// | ] +// |} +// """.stripMargin +// +// object Parser extends RequestParser +// +// val js = Json.parse(strJs) +// +// val q = Parser.toQuery(js) +// +// val queryResultLs = Await.result(s2.getEdges(q), 1 seconds) +// +// val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs) +// +// (resultJs \ "size").as[Int] must_== 500 +// +// (0 to 5) foreach { _ => +// duration("toSimpleVertexArrJson withScore=false org") { +// (0 to 1000) foreach { _ => +// PostProcess.toSimpleVertexArrJsonOrg(queryResultLs, Nil) +// } +// } +// +// duration("toSimpleVertexArrJson withScore=false new") { +// (0 to 1000) foreach { _ => +// PostProcess.toSimpleVertexArrJson(queryResultLs, Nil) +// } +// } +// } +// +// (resultJs \ "size").as[Int] must_== 500 +// } +// } +} diff --git a/s2rest_play/test/controllers/AdminControllerSpec.scala b/s2rest_play/test/controllers/AdminControllerSpec.scala new file mode 100644 index 00000000..e41fccb0 --- /dev/null +++ b/s2rest_play/test/controllers/AdminControllerSpec.scala @@ -0,0 +1,27 @@ +package controllers + +import com.kakao.s2graph.core.mysqls.Label +import play.api.http.HeaderNames +import play.api.test.Helpers._ +import play.api.test.{FakeApplication, FakeRequest} + +import scala.concurrent.Await + +/** + * Created by mojo22jojo(hyunsung.jo@gmail.com) on 15. 10. 13.. + */ +class AdminControllerSpec extends SpecCommon { + init() + "EdgeControllerSpec" should { + "update htable" in { + running(FakeApplication()) { + val insertUrl = s"/graphs/updateHTable/$testLabelName/$newHTableName" + + val req = FakeRequest("POST", insertUrl).withBody("").withHeaders(HeaderNames.CONTENT_TYPE -> "text/plain") + + Await.result(route(req).get, HTTP_REQ_WAITING_TIME) + Label.findByName(testLabelName, useCache = true).get.hTableName mustEqual newHTableName + } + } + } +} diff --git a/s2rest_play/test/controllers/BasicCrudSpec.scala b/s2rest_play/test/controllers/BasicCrudSpec.scala new file mode 100644 index 00000000..f4f11b45 --- /dev/null +++ b/s2rest_play/test/controllers/BasicCrudSpec.scala @@ -0,0 +1,251 @@ +package controllers + +import com.kakao.s2graph.core.Management +import com.kakao.s2graph.core.mysqls._ + +//import com.kakao.s2graph.core.models._ + +import play.api.libs.json._ +import play.api.test.Helpers._ +import play.api.test.{FakeApplication, FakeRequest} + +import scala.concurrent.Await + + +class BasicCrudSpec extends SpecCommon { + sequential + + var seed = 0 + def runTC(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = { + for { + labelName <- List(testLabelName, testLabelName2) + i <- 0 until NUM_OF_EACH_TEST + } { + seed += 1 +// val srcId = ((tcNum * 1000) + i).toString +// val tgtId = if (labelName == testLabelName) s"${srcId + 1000 + i}" else s"${srcId + 1000 + i}abc" + val srcId = seed.toString + val tgtId = srcId + + val maxTs = opWithProps.map(t => t._1).max + + /** insert edges */ + println(s"---- TC${tcNum}_init ----") + val bulkEdge = (for ((ts, op, props) <- opWithProps) yield { + List(ts, op, "e", srcId, tgtId, labelName, props).mkString("\t") + }).mkString("\n") + + val req = EdgeController.mutateAndPublish(bulkEdge, withWait = true) + val res = Await.result(req, HTTP_REQ_WAITING_TIME) + + res.header.status must equalTo(200) + + println(s"---- TC${tcNum}_init ----") +// Thread.sleep(100) + + for { + label <- Label.findByName(labelName) + direction <- List("out", "in") + cacheTTL <- List(-1L) + } { + val (serviceName, columnName, id, otherId) = direction match { + case "out" => (label.srcService.serviceName, label.srcColumn.columnName, srcId, tgtId) + case "in" => (label.tgtService.serviceName, label.tgtColumn.columnName, tgtId, srcId) + } + val qId = if (labelName == testLabelName) id else "\"" + id + "\"" + val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL) + val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(query)).get + val jsResult = commonCheck(ret) + + val results = jsResult \ "results" + val deegrees = (jsResult \ "degrees").as[List[JsObject]] + val propsLs = (results \\ "props").seq + (deegrees.head \ LabelMeta.degree.name).as[Int] must equalTo(1) + + val from = (results \\ "from").seq.last.toString.replaceAll("\"", "") + val to = (results \\ "to").seq.last.toString.replaceAll("\"", "") + + from must equalTo(id.toString) + to must equalTo(otherId.toString) +// (results \\ "_timestamp").seq.last.as[Long] must equalTo(maxTs) + for ((key, expectedVal) <- expected) { + propsLs.last.as[JsObject].keys.contains(key) must equalTo(true) + (propsLs.last \ key).toString must equalTo(expectedVal) + } + Await.result(ret, HTTP_REQ_WAITING_TIME) + } + } + } + + init() + "Basic Crud " should { + "tc1" in { + running(FakeApplication()) { + + var tcNum = 0 + var tcString = "" + var bulkQueries = List.empty[(Long, String, String)] + var expected = Map.empty[String, String] + + tcNum = 7 + tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test " + bulkQueries = List( + (t1, "insert", "{\"time\": 10}"), + (t2, "delete", ""), + (t3, "insert", "{\"time\": 10, \"weight\": 20}")) + expected = Map("time" -> "10", "weight" -> "20") + + runTC(tcNum, tcString, bulkQueries, expected) + + tcNum = 8 + tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test " + bulkQueries = List( + (t1, "insert", "{\"time\": 10}"), + (t3, "insert", "{\"time\": 10, \"weight\": 20}"), + (t2, "delete", "")) + expected = Map("time" -> "10", "weight" -> "20") + + runTC(tcNum, tcString, bulkQueries, expected) + + tcNum = 9 + tcString = "[t3 -> t2 -> t1 test case] insert(t3) delete(t2) insert(t1) test " + bulkQueries = List( + (t3, "insert", "{\"time\": 10, \"weight\": 20}"), + (t2, "delete", ""), + (t1, "insert", "{\"time\": 10}")) + expected = Map("time" -> "10", "weight" -> "20") + + runTC(tcNum, tcString, bulkQueries, expected) + + tcNum = 10 + tcString = "[t3 -> t1 -> t2 test case] insert(t3) insert(t1) delete(t2) test " + bulkQueries = List( + (t3, "insert", "{\"time\": 10, \"weight\": 20}"), + (t1, "insert", "{\"time\": 10}"), + (t2, "delete", "")) + expected = Map("time" -> "10", "weight" -> "20") + + runTC(tcNum, tcString, bulkQueries, expected) + + tcNum = 11 + tcString = "[t2 -> t1 -> t3 test case] delete(t2) insert(t1) insert(t3) test" + bulkQueries = List( + (t2, "delete", ""), + (t1, "insert", "{\"time\": 10}"), + (t3, "insert", "{\"time\": 10, \"weight\": 20}")) + expected = Map("time" -> "10", "weight" -> "20") + + runTC(tcNum, tcString, bulkQueries, expected) + + tcNum = 12 + tcString = "[t2 -> t3 -> t1 test case] delete(t2) insert(t3) insert(t1) test " + bulkQueries = List( + (t2, "delete", ""), + (t3, "insert", "{\"time\": 10, \"weight\": 20}"), + (t1, "insert", "{\"time\": 10}")) + expected = Map("time" -> "10", "weight" -> "20") + + runTC(tcNum, tcString, bulkQueries, expected) + + tcNum = 13 + tcString = "[t1 -> t2 -> t3 test case] update(t1) delete(t2) update(t3) test " + bulkQueries = List( + (t1, "update", "{\"time\": 10}"), + (t2, "delete", ""), + (t3, "update", "{\"time\": 10, \"weight\": 20}")) + expected = Map("time" -> "10", "weight" -> "20") + + runTC(tcNum, tcString, bulkQueries, expected) + tcNum = 14 + tcString = "[t1 -> t3 -> t2 test case] update(t1) update(t3) delete(t2) test " + bulkQueries = List( + (t1, "update", "{\"time\": 10}"), + (t3, "update", "{\"time\": 10, \"weight\": 20}"), + (t2, "delete", "")) + expected = Map("time" -> "10", "weight" -> "20") + + runTC(tcNum, tcString, bulkQueries, expected) + tcNum = 15 + tcString = "[t2 -> t1 -> t3 test case] delete(t2) update(t1) update(t3) test " + bulkQueries = List( + (t2, "delete", ""), + (t1, "update", "{\"time\": 10}"), + (t3, "update", "{\"time\": 10, \"weight\": 20}")) + expected = Map("time" -> "10", "weight" -> "20") + + runTC(tcNum, tcString, bulkQueries, expected) + tcNum = 16 + tcString = "[t2 -> t3 -> t1 test case] delete(t2) update(t3) update(t1) test" + bulkQueries = List( + (t2, "delete", ""), + (t3, "update", "{\"time\": 10, \"weight\": 20}"), + (t1, "update", "{\"time\": 10}")) + expected = Map("time" -> "10", "weight" -> "20") + + runTC(tcNum, tcString, bulkQueries, expected) + tcNum = 17 + tcString = "[t3 -> t2 -> t1 test case] update(t3) delete(t2) update(t1) test " + bulkQueries = List( + (t3, "update", "{\"time\": 10, \"weight\": 20}"), + (t2, "delete", ""), + (t1, "update", "{\"time\": 10}")) + expected = Map("time" -> "10", "weight" -> "20") + + runTC(tcNum, tcString, bulkQueries, expected) + tcNum = 18 + tcString = "[t3 -> t1 -> t2 test case] update(t3) update(t1) delete(t2) test " + bulkQueries = List( + (t3, "update", "{\"time\": 10, \"weight\": 20}"), + (t1, "update", "{\"time\": 10}"), + (t2, "delete", "")) + expected = Map("time" -> "10", "weight" -> "20") + + runTC(tcNum, tcString, bulkQueries, expected) + + tcNum = 19 + tcString = "[t5 -> t1 -> t3 -> t2 -> t4 test case] update(t5) insert(t1) insert(t3) delete(t2) update(t4) test " + bulkQueries = List( + (t5, "update", "{\"is_blocked\": true}"), + (t1, "insert", "{\"is_hidden\": false}"), + (t3, "insert", "{\"is_hidden\": false, \"weight\": 10}"), + (t2, "delete", ""), + (t4, "update", "{\"time\": 1, \"weight\": -10}")) + expected = Map("time" -> "1", "weight" -> "-10", "is_hidden" -> "false", "is_blocked" -> "true") + + runTC(tcNum, tcString, bulkQueries, expected) + true + } + } + } + + "toLogString" in { + running(FakeApplication()) { + val bulkQueries = List( + ("1445240543366", "update", "{\"is_blocked\":true}"), + ("1445240543362", "insert", "{\"is_hidden\":false}"), + ("1445240543364", "insert", "{\"is_hidden\":false,\"weight\":10}"), + ("1445240543363", "delete", "{}"), + ("1445240543365", "update", "{\"time\":1, \"weight\":-10}")) + + val (srcId, tgtId, labelName) = ("1", "2", testLabelName) + + val bulkEdge = (for ((ts, op, props) <- bulkQueries) yield { + Management.toEdge(ts.toLong, op, srcId, tgtId, labelName, "out", props).toLogString + }).mkString("\n") + + val expected = Seq( + Seq("1445240543366", "update", "e", "1", "2", "s2graph_label_test", "{\"is_blocked\":true}"), + Seq("1445240543362", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false}"), + Seq("1445240543364", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false,\"weight\":10}"), + Seq("1445240543363", "delete", "e", "1", "2", "s2graph_label_test"), + Seq("1445240543365", "update", "e", "1", "2", "s2graph_label_test", "{\"time\":1,\"weight\":-10}") + ).map(_.mkString("\t")).mkString("\n") + + bulkEdge must equalTo(expected) + + true + } + } +} + + diff --git a/s2rest_play/test/controllers/EdgeControllerSpec.scala b/s2rest_play/test/controllers/EdgeControllerSpec.scala new file mode 100644 index 00000000..e76404ab --- /dev/null +++ b/s2rest_play/test/controllers/EdgeControllerSpec.scala @@ -0,0 +1,22 @@ +package controllers + +import play.api.http.HeaderNames +import play.api.test.{FakeApplication, FakeRequest, PlaySpecification, WithApplication} +import play.api.{Application => PlayApplication} + +/** + * Created by hsleep(honeysleep@gmail.com) on 15. 9. 1.. + */ +class EdgeControllerSpec extends PlaySpecification { +// "EdgeControllerSpec" should { +// implicit val app = FakeApplication() +// +// "bad request invalid json" in new WithApplication(app) { +// val insertUrl = "http://localhost:9000/graphs/edges/insert" +// val req = FakeRequest("POST", insertUrl).withBody("").withHeaders(HeaderNames.CONTENT_TYPE -> "application/json") +// val result = EdgeController.inserts().apply(req).run +// +// status(result) must_== BAD_REQUEST +// } +// } +} diff --git a/s2rest_play/test/controllers/QueryCacheSpec.scala b/s2rest_play/test/controllers/QueryCacheSpec.scala new file mode 100644 index 00000000..7d91aa98 --- /dev/null +++ b/s2rest_play/test/controllers/QueryCacheSpec.scala @@ -0,0 +1,89 @@ +//package test.controllers +// +////import com.kakao.s2graph.core.models._ +// +//import controllers.EdgeController +//import play.api.libs.json._ +//import play.api.test.Helpers._ +//import play.api.test.{FakeApplication, FakeRequest} +// +//class QueryCacheSpec extends SpecCommon { +// init() +// +// "cache test" should { +// def queryWithTTL(id: Int, cacheTTL: Long) = Json.parse( s""" +// { "srcVertices": [ +// { "serviceName": "${testServiceName}", +// "columnName": "${testColumnName}", +// "id": ${id} +// }], +// "steps": [[ { +// "label": "${testLabelName}", +// "direction": "out", +// "offset": 0, +// "limit": 10, +// "cacheTTL": ${cacheTTL}, +// "scoring": {"weight": 1} }]] +// }""") +// +// def getEdges(queryJson: JsValue): JsValue = { +// var ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get +// contentAsJson(ret) +// } +// +// // init +// running(FakeApplication()) { +// // insert bulk and wait .. +// val bulkEdges: String = Seq( +// Seq("1", "insert", "e", "0", "2", "s2graph_label_test", "{}").mkString("\t"), +// Seq("1", "insert", "e", "1", "2", "s2graph_label_test", "{}").mkString("\t") +// ).mkString("\n") +// +// val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) +// Thread.sleep(asyncFlushInterval) +// } +// +// "tc1: query with {id: 0, ttl: 1000}" in { +// running(FakeApplication()) { +// var jsRslt = getEdges(queryWithTTL(0, 1000)) +// var cacheRemain = (jsRslt \\ "cacheRemain").head +// cacheRemain.as[Int] must greaterThan(500) +// +// // get edges from cache after wait 500ms +// Thread.sleep(500) +// val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryWithTTL(0, 1000))).get +// jsRslt = contentAsJson(ret) +// cacheRemain = (jsRslt \\ "cacheRemain").head +// cacheRemain.as[Int] must lessThan(500) +// } +// } +// +// "tc2: query with {id: 1, ttl: 3000}" in { +// running(FakeApplication()) { +// var jsRslt = getEdges(queryWithTTL(1, 3000)) +// var cacheRemain = (jsRslt \\ "cacheRemain").head +// // before update: is_blocked is false +// (jsRslt \\ "is_blocked").head must equalTo(JsBoolean(false)) +// +// val bulkEdges = Seq( +// Seq("2", "update", "e", "0", "2", "s2graph_label_test", "{\"is_blocked\": true}").mkString("\t"), +// Seq("2", "update", "e", "1", "2", "s2graph_label_test", "{\"is_blocked\": true}").mkString("\t") +// ).mkString("\n") +// +// // update edges with {is_blocked: true} +// jsRslt = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) +// +// Thread.sleep(asyncFlushInterval) +// +// // prop 'is_blocked' still false, cause queryResult on cache +// jsRslt = getEdges(queryWithTTL(1, 3000)) +// (jsRslt \\ "is_blocked").head must equalTo(JsBoolean(false)) +// +// // after wait 3000ms prop 'is_blocked' is updated to true, cache cleared +// Thread.sleep(3000) +// jsRslt = getEdges(queryWithTTL(1, 3000)) +// (jsRslt \\ "is_blocked").head must equalTo(JsBoolean(true)) +// } +// } +// } +//} diff --git a/s2rest_play/test/controllers/QuerySpec.scala b/s2rest_play/test/controllers/QuerySpec.scala new file mode 100644 index 00000000..f6381bc0 --- /dev/null +++ b/s2rest_play/test/controllers/QuerySpec.scala @@ -0,0 +1,604 @@ +package controllers + +import play.api.libs.json._ +import play.api.test.{FakeApplication, FakeRequest, PlaySpecification} +import play.api.{Application => PlayApplication} + +import scala.concurrent.Await + +class QuerySpec extends SpecCommon with PlaySpecification { + + import Helper._ + + implicit val app = FakeApplication() + + init() + + "query test" should { + running(FakeApplication()) { + + // insert bulk and wait .. + val bulkEdges: String = Seq( + edge"1000 insert e 0 1 $testLabelName"($(weight = 40, is_hidden = true)), + edge"2000 insert e 0 2 $testLabelName"($(weight = 30, is_hidden = false)), + edge"3000 insert e 2 0 $testLabelName"($(weight = 20)), + edge"4000 insert e 2 1 $testLabelName"($(weight = 10)), + edge"3000 insert e 10 20 $testLabelName"($(weight = 20)), + edge"4000 insert e 20 20 $testLabelName"($(weight = 10)), + edge"1 insert e -1 1000 $testLabelName", + edge"1 insert e -1 2000 $testLabelName", + edge"1 insert e -1 3000 $testLabelName", + edge"1 insert e 1000 10000 $testLabelName", + edge"1 insert e 1000 11000 $testLabelName", + edge"1 insert e 2000 11000 $testLabelName", + edge"1 insert e 2000 12000 $testLabelName", + edge"1 insert e 3000 12000 $testLabelName", + edge"1 insert e 3000 13000 $testLabelName", + edge"1 insert e 10000 100000 $testLabelName", + edge"2 insert e 11000 200000 $testLabelName", + edge"3 insert e 12000 300000 $testLabelName").mkString("\n") + + val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) + } + + def queryParents(id: Long) = Json.parse( s""" + { + "returnTree": true, + "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${testLabelName}", + "direction": "out", + "offset": 0, + "limit": 2 + } + ],[{ + "label": "${testLabelName}", + "direction": "in", + "offset": 0, + "limit": -1 + } + ]] + }""".stripMargin) + + def queryExclude(id: Int) = Json.parse( s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${testLabelName}", + "direction": "out", + "offset": 0, + "limit": 2 + }, + { + "label": "${testLabelName}", + "direction": "in", + "offset": 0, + "limit": 2, + "exclude": true + } + ]] + }""") + + def queryTransform(id: Int, transforms: String) = Json.parse( s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${testLabelName}", + "direction": "out", + "offset": 0, + "transform": $transforms + } + ]] + }""") + + def queryWhere(id: Int, where: String) = Json.parse( s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${testLabelName}", + "direction": "out", + "offset": 0, + "limit": 100, + "where": "${where}" + } + ]] + }""") + + def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse( s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${testLabelName}", + "direction": "out", + "offset": $offset, + "limit": $limit, + "_to": $to + } + ]] + } + """) + + def querySingle(id: Int, offset: Int = 0, limit: Int = 100) = Json.parse( s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + [ { + "label": "${testLabelName}", + "direction": "out", + "offset": $offset, + "limit": $limit + } + ]] + } + """) + + def queryWithSampling(id: Int, sample: Int) = Json.parse( s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + { + "step": [{ + "label": "${testLabelName}", + "direction": "out", + "offset": 0, + "limit": 100, + "sample": ${sample} + }] + } + ] + }""") + + + def twoStepQueryWithSampling(id: Int, sample: Int) = Json.parse( s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + { + "step": [{ + "label": "${testLabelName}", + "direction": "out", + "offset": 0, + "limit": 100, + "sample": ${sample} + }] + }, + { + "step": [{ + "label": "${testLabelName}", + "direction": "out", + "offset": 0, + "limit": 100, + "sample": ${sample} + }] + } + ] + }""") + + def twoQueryWithSampling(id: Int, sample: Int) = Json.parse( s""" + { "srcVertices": [ + { "serviceName": "${testServiceName}", + "columnName": "${testColumnName}", + "id": ${id} + }], + "steps": [ + { + "step": [{ + "label": "${testLabelName}", + "direction": "out", + "offset": 0, + "limit": 50, + "sample": ${sample} + }, + { + "label": "${testLabelName2}", + "direction": "out", + "offset": 0, + "limit": 50 + }] + } + ] + }""") + + def queryUnion(id: Int, size: Int) = JsArray(List.tabulate(size)(_ => querySingle(id))) + + def queryGroupBy(id: Int, props: Seq[String]): JsValue = { + Json.obj( + "groupBy" -> props, + "srcVertices" -> Json.arr( + Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) + ), + "steps" -> Json.arr( + Json.obj( + "step" -> Json.arr( + Json.obj( + "label" -> testLabelName + ) + ) + ) + ) + ) + } + + def queryScore(id: Int, scoring: Map[String, Int]): JsValue = { + val q = Json.obj( + "srcVertices" -> Json.arr( + Json.obj( + "serviceName" -> testServiceName, + "columnName" -> testColumnName, + "id" -> id + ) + ), + "steps" -> Json.arr( + Json.obj( + "step" -> Json.arr( + Json.obj( + "label" -> testLabelName, + "scoring" -> scoring + ) + ) + ) + ) + ) + println(q) + q + } + + def queryOrderBy(id: Int, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = { + Json.obj( + "orderBy" -> props, + "srcVertices" -> Json.arr( + Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) + ), + "steps" -> Json.arr( + Json.obj( + "step" -> Json.arr( + Json.obj( + "label" -> testLabelName, + "scoring" -> scoring + ) + ) + ) + ) + ) + } + + def getEdges(queryJson: JsValue): JsValue = { + val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get + contentAsJson(ret) + } + + def queryIndex(ids: Seq[Int], indexName: String) = { + val $from = $a( + $(serviceName = testServiceName, + columnName = testColumnName, + ids = ids)) + + val $step = $a($(label = testLabelName, index = indexName)) + val $steps = $a($(step = $step)) + + val js = $(withScore = false, srcVertices = $from, steps = $steps).toJson + js + } + + def queryDuration(ids: Seq[Int], from: Int, to: Int) = { + val $from = $a( + $(serviceName = testServiceName, + columnName = testColumnName, + ids = ids)) + + val $step = $a($( + label = testLabelName, direction = "out", offset = 0, limit = 100, + duration = $(from = from, to = to))) + + val $steps = $a($(step = $step)) + + $(srcVertices = $from, steps = $steps).toJson + } + + "union query" in { + running(FakeApplication()) { + var result = getEdges(queryUnion(0, 2)) + result.as[List[JsValue]].size must equalTo(2) + + result = getEdges(queryUnion(0, 3)) + result.as[List[JsValue]].size must equalTo(3) + + result = getEdges(queryUnion(0, 4)) + result.as[List[JsValue]].size must equalTo(4) + + result = getEdges(queryUnion(0, 5)) + result.as[List[JsValue]].size must equalTo(5) + + val union = result.as[List[JsValue]].head + val single = getEdges(querySingle(0)) + + (union \\ "from").map(_.toString).sorted must equalTo((single \\ "from").map(_.toString).sorted) + (union \\ "to").map(_.toString).sorted must equalTo((single \\ "to").map(_.toString).sorted) + (union \\ "weight").map(_.toString).sorted must equalTo((single \\ "weight").map(_.toString).sorted) + } + } + + "get edge with where condition" in { + running(FakeApplication()) { + var result = getEdges(queryWhere(0, "is_hidden=false and _from in (-1, 0)")) + (result \ "results").as[List[JsValue]].size must equalTo(1) + + result = getEdges(queryWhere(0, "is_hidden=true and _to in (1)")) + (result \ "results").as[List[JsValue]].size must equalTo(1) + + result = getEdges(queryWhere(0, "_from=0")) + (result \ "results").as[List[JsValue]].size must equalTo(2) + + result = getEdges(queryWhere(2, "_from=2 or weight in (-1)")) + (result \ "results").as[List[JsValue]].size must equalTo(2) + + result = getEdges(queryWhere(2, "_from=2 and weight in (10, 20)")) + (result \ "results").as[List[JsValue]].size must equalTo(2) + } + } + + "get edge exclude" in { + running(FakeApplication()) { + val result = getEdges(queryExclude(0)) + (result \ "results").as[List[JsValue]].size must equalTo(1) + } + } + + "get edge groupBy property" in { + running(FakeApplication()) { + val result = getEdges(queryGroupBy(0, Seq("weight"))) + (result \ "size").as[Int] must_== 2 + val weights = (result \\ "groupBy").map { js => + (js \ "weight").as[Int] + } + weights must contain(exactly(30, 40)) + weights must not contain (10) + } + } + + "edge transform " in { + running(FakeApplication()) { + var result = getEdges(queryTransform(0, "[[\"_to\"]]")) + (result \ "results").as[List[JsValue]].size must equalTo(2) + + result = getEdges(queryTransform(0, "[[\"weight\"]]")) + (result \\ "to").map(_.toString).sorted must equalTo((result \\ "weight").map(_.toString).sorted) + + result = getEdges(queryTransform(0, "[[\"_from\"]]")) + val results = (result \ "results").as[JsValue] + (result \\ "to").map(_.toString).sorted must equalTo((results \\ "from").map(_.toString).sorted) + } + } + + "index" in { + running(FakeApplication()) { + // weight order + var result = getEdges(queryIndex(Seq(0), "idx_1")) + ((result \ "results").as[List[JsValue]].head \\ "weight").head must equalTo(JsNumber(40)) + + // timestamp order + result = getEdges(queryIndex(Seq(0), "idx_2")) + ((result \ "results").as[List[JsValue]].head \\ "weight").head must equalTo(JsNumber(30)) + } + } + + "checkEdges" in { + running(FakeApplication()) { + val json = Json.parse( s""" + [{"from": 0, "to": 1, "label": "$testLabelName"}, + {"from": 0, "to": 2, "label": "$testLabelName"}] + """) + + def checkEdges(queryJson: JsValue): JsValue = { + val ret = route(FakeRequest(POST, "/graphs/checkEdges").withJsonBody(queryJson)).get + contentAsJson(ret) + } + + val res = checkEdges(json) + val typeRes = res.isInstanceOf[JsArray] + typeRes must equalTo(true) + + val fst = res.as[Seq[JsValue]].head \ "to" + fst.as[Int] must equalTo(1) + + val snd = res.as[Seq[JsValue]].last \ "to" + snd.as[Int] must equalTo(2) + } + } + + "duration" in { + running(FakeApplication()) { + // get all + var result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000)) + (result \ "results").as[List[JsValue]].size must equalTo(4) + + // inclusive, exclusive + result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000)) + (result \ "results").as[List[JsValue]].size must equalTo(3) + + result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000)) + (result \ "results").as[List[JsValue]].size must equalTo(1) + + val bulkEdges: String = Seq( + edge"1001 insert e 0 1 $testLabelName"($(weight = 10, is_hidden = true)), + edge"2002 insert e 0 2 $testLabelName"($(weight = 20, is_hidden = false)), + edge"3003 insert e 2 0 $testLabelName"($(weight = 30)), + edge"4004 insert e 2 1 $testLabelName"($(weight = 40)) + ).mkString("\n") + + val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) + // duration test after udpate + // get all + result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000)) + (result \ "results").as[List[JsValue]].size must equalTo(4) + + // inclusive, exclusive + result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000)) + (result \ "results").as[List[JsValue]].size must equalTo(3) + + result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000)) + (result \ "results").as[List[JsValue]].size must equalTo(1) + true + } + } + + "returnTree" in { + running(FakeApplication()) { + val src = 100 + val tgt = 200 + val labelName = testLabelName + + val bulkEdges: String = Seq( + edge"1001 insert e $src $tgt $labelName" + ).mkString("\n") + + val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) + + val result = getEdges(queryParents(src)) + + val parents = (result \ "results").as[Seq[JsValue]] + val ret = parents.forall { edge => (edge \ "parents").as[Seq[JsValue]].size == 1 } + ret must equalTo(true) + } + } + + "pagination and _to" in { + running(FakeApplication()) { + val src = System.currentTimeMillis().toInt + val labelName = testLabelName + val bulkEdges: String = Seq( + edge"1001 insert e $src 1 $labelName"($(weight = 10, is_hidden = true)), + edge"2002 insert e $src 2 $labelName"($(weight = 20, is_hidden = false)), + edge"3003 insert e $src 3 $labelName"($(weight = 30)), + edge"4004 insert e $src 4 $labelName"($(weight = 40)) + ).mkString("\n") + + val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) + + var result = getEdges(querySingle(src, offset = 0, limit = 2)) + println(result) + var edges = (result \ "results").as[List[JsValue]] + edges.size must equalTo(2) + (edges(0) \ "to").as[Long] must beEqualTo(4) + (edges(1) \ "to").as[Long] must beEqualTo(3) + + result = getEdges(querySingle(src, offset = 1, limit = 2)) + println(result) + edges = (result \ "results").as[List[JsValue]] + edges.size must equalTo(2) + (edges(0) \ "to").as[Long] must beEqualTo(3) + (edges(1) \ "to").as[Long] must beEqualTo(2) + + result = getEdges(querySingleWithTo(src, offset = 0, limit = -1, to = 1)) + println(result) + edges = (result \ "results").as[List[JsValue]] + edges.size must equalTo(1) + } + } + + "orderBy" >> { + running(FakeApplication()) { + // insert test set + val bulkEdges: String = Seq( + edge"1001 insert e 0 1 $testLabelName"($(weight = 10, is_hidden = true)), + edge"2002 insert e 0 2 $testLabelName"($(weight = 20, is_hidden = false)), + edge"3003 insert e 2 0 $testLabelName"($(weight = 30)), + edge"4004 insert e 2 1 $testLabelName"($(weight = 40)) + ).mkString("\n") + contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) + + // get edges + val edges = getEdges(queryScore(0, Map("weight" -> 1))) + val orderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC")))) + val ascOrderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC")))) + + println(edges) + println(orderByScore) + println(ascOrderByScore) + + val edgesTo = edges \ "results" \\ "to" + val orderByTo = orderByScore \ "results" \\ "to" + val ascOrderByTo = ascOrderByScore \ "results" \\ "to" + + edgesTo must_== Seq(JsNumber(2), JsNumber(1)) + edgesTo must_== orderByTo + ascOrderByTo must_== Seq(JsNumber(1), JsNumber(2)) + edgesTo.reverse must_== ascOrderByTo + } + } + + "query with sampling" in { + running(FakeApplication()) { + val sampleSize = 2 + val testId = 22 + val bulkEdges = Seq( + edge"1442985659166 insert e $testId 122 $testLabelName", + edge"1442985659166 insert e $testId 222 $testLabelName", + edge"1442985659166 insert e $testId 322 $testLabelName", + + edge"1442985659166 insert e $testId 922 $testLabelName2", + edge"1442985659166 insert e $testId 222 $testLabelName2", + edge"1442985659166 insert e $testId 322 $testLabelName2", + + edge"1442985659166 insert e 122 1122 $testLabelName", + edge"1442985659166 insert e 122 1222 $testLabelName", + edge"1442985659166 insert e 122 1322 $testLabelName", + edge"1442985659166 insert e 222 2122 $testLabelName", + edge"1442985659166 insert e 222 2222 $testLabelName", + edge"1442985659166 insert e 222 2322 $testLabelName", + edge"1442985659166 insert e 322 3122 $testLabelName", + edge"1442985659166 insert e 322 3222 $testLabelName", + edge"1442985659166 insert e 322 3322 $testLabelName" + ) + + val req = FakeRequest(POST, "/graphs/edges/bulk").withBody(bulkEdges.mkString("\n")) + Await.result(route(req).get, HTTP_REQ_WAITING_TIME) + + Thread.sleep(asyncFlushInterval) + + + val result1 = getEdges(queryWithSampling(testId, sampleSize)) + println(Json.toJson(result1)) + (result1 \ "results").as[List[JsValue]].size must equalTo(scala.math.min(sampleSize, bulkEdges.size)) + + val result2 = getEdges(twoStepQueryWithSampling(testId, sampleSize)) + println(Json.toJson(result2)) + (result2 \ "results").as[List[JsValue]].size must equalTo(scala.math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size)) + + val result3 = getEdges(twoQueryWithSampling(testId, sampleSize)) + println(Json.toJson(result3)) + (result3 \ "results").as[List[JsValue]].size must equalTo(sampleSize + 3) // edges in testLabelName2 = 3 + } + } + } +} diff --git a/s2rest_play/test/controllers/SpecCommon.scala b/s2rest_play/test/controllers/SpecCommon.scala new file mode 100644 index 00000000..93648ce9 --- /dev/null +++ b/s2rest_play/test/controllers/SpecCommon.scala @@ -0,0 +1,365 @@ +package controllers + +import com.kakao.s2graph.core._ +import com.kakao.s2graph.core.mysqls._ +import org.specs2.mutable.Specification +import play.api.libs.json._ +import play.api.test.FakeApplication +import play.api.test.Helpers._ + +import scala.concurrent.Future +import scala.concurrent.duration._ +import scala.util.Random + +trait SpecCommon extends Specification { + sequential + object Helper { + + import org.json4s.native.Serialization + + type KV = Map[String, Any] + + import scala.language.dynamics + + def $aa[T](args: T*) = List($a(args: _ *)) + + def $a[T](args: T*) = args.toList + + object $ extends Dynamic { + def applyDynamicNamed(name: String)(args: (String, Any)*): Map[String, Any] = args.toMap + } + + implicit class anyMapOps(map: Map[String, Any]) { + def toJson: JsValue = { + val js = Serialization.write(map)(org.json4s.DefaultFormats) + Json.parse(js) + } + } + + implicit class S2Context(val sc: StringContext) { + def edge(args: Any*)(implicit map: Map[String, Any] = Map.empty): String = { + val parts = sc.s(args: _*).split("\\s") + assert(parts.length == 6) + (parts.toList :+ map.toJson.toString).mkString("\t") + } + } + + } + + val curTime = System.currentTimeMillis + val t1 = curTime + 0 + val t2 = curTime + 1 + val t3 = curTime + 2 + val t4 = curTime + 3 + val t5 = curTime + 4 + + protected val testServiceName = "s2graph" + protected val testLabelName = "s2graph_label_test" + protected val testLabelName2 = "s2graph_label_test_2" + protected val testLabelNameV1 = "s2graph_label_test_v1" + protected val testLabelNameWeak = "s2graph_label_test_weak" + protected val testColumnName = "user_id_test" + protected val testColumnType = "long" + protected val testTgtColumnName = "item_id_test" + protected val testHTableName = "test-htable" + protected val newHTableName = "new-htable" + + val NUM_OF_EACH_TEST = 100 + val HTTP_REQ_WAITING_TIME = Duration(300, SECONDS) + val asyncFlushInterval = 100 + + val createService = s"""{"serviceName" : "$testServiceName"}""" + val testLabelNameCreate = + s""" + { + "label": "$testLabelName", + "srcServiceName": "$testServiceName", + "srcColumnName": "$testColumnName", + "srcColumnType": "long", + "tgtServiceName": "$testServiceName", + "tgtColumnName": "$testColumnName", + "tgtColumnType": "long", + "indices": [ + {"name": "idx_1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]}, + {"name": "idx_2", "propNames": ["_timestamp"]} + ], + "props": [ + { + "name": "time", + "dataType": "long", + "defaultValue": 0 + }, + { + "name": "weight", + "dataType": "long", + "defaultValue": 0 + }, + { + "name": "is_hidden", + "dataType": "boolean", + "defaultValue": false + }, + { + "name": "is_blocked", + "dataType": "boolean", + "defaultValue": false + } + ], + "consistencyLevel": "strong", + "schemaVersion": "v2", + "compressionAlgorithm": "gz", + "hTableName": "$testHTableName" + }""" + + val testLabelName2Create = + s""" + { + "label": "$testLabelName2", + "srcServiceName": "$testServiceName", + "srcColumnName": "$testColumnName", + "srcColumnType": "long", + "tgtServiceName": "$testServiceName", + "tgtColumnName": "$testTgtColumnName", + "tgtColumnType": "string", + "indices": [{"name": "idx_1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}], + "props": [ + { + "name": "time", + "dataType": "long", + "defaultValue": 0 + }, + { + "name": "weight", + "dataType": "long", + "defaultValue": 0 + }, + { + "name": "is_hidden", + "dataType": "boolean", + "defaultValue": false + }, + { + "name": "is_blocked", + "dataType": "boolean", + "defaultValue": false + } + ], + "consistencyLevel": "strong", + "isDirected": false, + "schemaVersion": "v3", + "compressionAlgorithm": "gz" + }""" + + val testLabelNameV1Create = + s""" + { + "label": "$testLabelNameV1", + "srcServiceName": "$testServiceName", + "srcColumnName": "$testColumnName", + "srcColumnType": "long", + "tgtServiceName": "$testServiceName", + "tgtColumnName": "${testTgtColumnName}_v1", + "tgtColumnType": "string", + "indices": [{"name": "idx_1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}], + "props": [ + { + "name": "time", + "dataType": "long", + "defaultValue": 0 + }, + { + "name": "weight", + "dataType": "long", + "defaultValue": 0 + }, + { + "name": "is_hidden", + "dataType": "boolean", + "defaultValue": false + }, + { + "name": "is_blocked", + "dataType": "boolean", + "defaultValue": false + } + ], + "consistencyLevel": "strong", + "isDirected": true, + "schemaVersion": "v1", + "compressionAlgorithm": "gz" + }""" + val testLabelNameWeakCreate = + s""" + { + "label": "$testLabelNameWeak", + "srcServiceName": "$testServiceName", + "srcColumnName": "$testColumnName", + "srcColumnType": "long", + "tgtServiceName": "$testServiceName", + "tgtColumnName": "$testTgtColumnName", + "tgtColumnType": "string", + "indices": [{"name": "idx_1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}], + "props": [ + { + "name": "time", + "dataType": "long", + "defaultValue": 0 + }, + { + "name": "weight", + "dataType": "long", + "defaultValue": 0 + }, + { + "name": "is_hidden", + "dataType": "boolean", + "defaultValue": false + }, + { + "name": "is_blocked", + "dataType": "boolean", + "defaultValue": false + } + ], + "consistencyLevel": "weak", + "isDirected": true, + "compressionAlgorithm": "gz" + }""" + + val vertexPropsKeys = List( + ("age", "int") + ) + + val createVertex = + s"""{ + "serviceName": "$testServiceName", + "columnName": "$testColumnName", + "columnType": "$testColumnType", + "props": [ + {"name": "is_active", "dataType": "boolean", "defaultValue": true}, + {"name": "phone_number", "dataType": "string", "defaultValue": "-"}, + {"name": "nickname", "dataType": "string", "defaultValue": ".."}, + {"name": "activity_score", "dataType": "float", "defaultValue": 0.0}, + {"name": "age", "dataType": "integer", "defaultValue": 0} + ] + }""" + + + val TS = System.currentTimeMillis() + + def queryJson(serviceName: String, columnName: String, labelName: String, id: String, dir: String, cacheTTL: Long = -1L) = { + val s = + s"""{ + "srcVertices": [ + { + "serviceName": "$serviceName", + "columnName": "$columnName", + "id": $id + } + ], + "steps": [ + [ + { + "label": "$labelName", + "direction": "$dir", + "offset": 0, + "limit": 10, + "cacheTTL": $cacheTTL + } + ] + ] + }""" + println(s) + Json.parse(s) + } + + def checkEdgeQueryJson(params: Seq[(String, String, String, String)]) = { + val arr = for { + (label, dir, from, to) <- params + } yield { + Json.obj("label" -> label, "direction" -> dir, "from" -> from, "to" -> to) + } + + val s = Json.toJson(arr) + println(s) + s + } + + def vertexQueryJson(serviceName: String, columnName: String, ids: Seq[Int]) = { + Json.parse( + s""" + |[ + |{"serviceName": "$serviceName", "columnName": "$columnName", "ids": [${ids.mkString(",")} + ]} + |] + """.stripMargin) + } + + def randomProps() = { + (for { + (propKey, propType) <- vertexPropsKeys + } yield { + propKey -> Random.nextInt(100) + }).toMap + } + + def vertexInsertsPayload(serviceName: String, columnName: String, ids: Seq[Int]): Seq[JsValue] = { + ids.map { id => + Json.obj("id" -> id, "props" -> randomProps, "timestamp" -> System.currentTimeMillis()) + } + } + + def commonCheck(rslt: Future[play.api.mvc.Result]): JsValue = { + status(rslt) must equalTo(OK) + contentType(rslt) must beSome.which(_ == "application/json") + val jsRslt = contentAsJson(rslt) + println("======") + println(jsRslt) + println("======") + jsRslt.as[JsObject].keys.contains("size") must equalTo(true) + (jsRslt \ "size").as[Int] must greaterThan(0) + jsRslt.as[JsObject].keys.contains("results") must equalTo(true) + val jsRsltsObj = jsRslt \ "results" + jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("from") must equalTo(true) + jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("to") must equalTo(true) + jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("_timestamp") must equalTo(true) + jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("props") must equalTo(true) + jsRslt + } + + def init() = { + running(FakeApplication()) { + println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") + Management.deleteService(testServiceName) + + // 1. createService + val result = AdminController.createServiceInner(Json.parse(createService)) + println(s">> Service created : $createService, $result") + + val labelNames = Map(testLabelName -> testLabelNameCreate, + testLabelName2 -> testLabelName2Create, + testLabelNameV1 -> testLabelNameV1Create, + testLabelNameWeak -> testLabelNameWeakCreate) + + for { + (labelName, create) <- labelNames + } { + Management.deleteLabel(labelName) + Label.findByName(labelName, useCache = false) match { + case None => + AdminController.createLabelInner(Json.parse(create)) + case Some(label) => + println(s">> Label already exist: $create, $label") + } + } + + // 5. create vertex + vertexPropsKeys.map { case (key, keyType) => + Management.addVertexProp(testServiceName, testColumnName, key, keyType) + } + + println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") + } + } +} + diff --git a/s2rest_play/test/controllers/StrongLabelDeleteSpec.scala b/s2rest_play/test/controllers/StrongLabelDeleteSpec.scala new file mode 100644 index 00000000..04ccc79e --- /dev/null +++ b/s2rest_play/test/controllers/StrongLabelDeleteSpec.scala @@ -0,0 +1,345 @@ +package controllers + +import java.util.concurrent.TimeUnit + + +import com.kakao.s2graph.core.utils.logger +import play.api.libs.json._ +import play.api.test.Helpers._ +import play.api.test.{FakeApplication, FakeRequest} + +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, Future} +import scala.util.Random + +class StrongLabelDeleteSpec extends SpecCommon { + init() +// implicit val timeout = Timeout(Duration(20, TimeUnit.MINUTES)) + + def bulkEdges(startTs: Int = 0) = Seq( + Seq(startTs + 1, "insert", "e", "0", "1", testLabelName2, s"""{"time": 10}""").mkString("\t"), + Seq(startTs + 2, "insert", "e", "0", "1", testLabelName2, s"""{"time": 11}""").mkString("\t"), + Seq(startTs + 3, "insert", "e", "0", "1", testLabelName2, s"""{"time": 12}""").mkString("\t"), + Seq(startTs + 4, "insert", "e", "0", "2", testLabelName2, s"""{"time": 10}""").mkString("\t"), + Seq(startTs + 5, "insert", "e", "10", "20", testLabelName2, s"""{"time": 10}""").mkString("\t"), + Seq(startTs + 6, "insert", "e", "10", "21", testLabelName2, s"""{"time": 11}""").mkString("\t"), + Seq(startTs + 7, "insert", "e", "11", "20", testLabelName2, s"""{"time": 12}""").mkString("\t"), + Seq(startTs + 8, "insert", "e", "12", "20", testLabelName2, s"""{"time": 13}""").mkString("\t") + ).mkString("\n") + + def query(id: Long, serviceName: String = testServiceName, columnName: String = testColumnName, + labelName: String = testLabelName2, direction: String = "out") = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "$serviceName", + "columnName": "$columnName", + "id": $id + }], + "steps": [ + [ { + "label": "$labelName", + "direction": "${direction}", + "offset": 0, + "limit": -1, + "duplicate": "raw" + } + ]] + }""") + + def getEdges(queryJson: JsValue): JsValue = { +// implicit val timeout = Timeout(Duration(20, TimeUnit.MINUTES)) + + val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get + contentAsJson(ret) + } + + def getDegree(jsValue: JsValue): Long = { + ((jsValue \ "degrees") \\ "_degree").headOption.map(_.as[Long]).getOrElse(0L) + } + + + "strong label delete test" should { + running(FakeApplication()) { + // insert bulk and wait .. + val edges = bulkEdges() + println(edges) + val jsResult = contentAsJson(EdgeController.mutateAndPublish(edges, withWait = true)) + } + + "test strong consistency select" in { + running(FakeApplication()) { + var result = getEdges(query(0)) + println(result) + (result \ "results").as[List[JsValue]].size must equalTo(2) + result = getEdges(query(10)) + println(result) + (result \ "results").as[List[JsValue]].size must equalTo(2) + true + } + } + + "test strong consistency duration. insert -> delete -> insert" in { + running(FakeApplication()) { + val ts0 = 1 + val ts1 = 2 + val ts2 = 3 + + val edges = Seq( + Seq(5, "insert", "edge", "-10", "-20", testLabelName2).mkString("\t"), + Seq(10, "delete", "edge", "-10", "-20", testLabelName2).mkString("\t"), + Seq(20, "insert", "edge", "-10", "-20", testLabelName2).mkString("\t") + ).mkString("\n") + + val jsResult = contentAsJson(EdgeController.mutateAndPublish(edges, withWait = true)) + + val result = getEdges(query(-10)) + + println(result) + + true + } + } + + "test strong consistency deleteAll" in { + running(FakeApplication()) { + + val deletedAt = 100 + var result = getEdges(query(20, direction = "in", columnName = testTgtColumnName)) + println(result) + (result \ "results").as[List[JsValue]].size must equalTo(3) + + val json = Json.arr(Json.obj("label" -> testLabelName2, + "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt)) + println(json) + contentAsString(EdgeController.deleteAllInner(json, withWait = true)) + + result = getEdges(query(11, direction = "out")) + println(result) + (result \ "results").as[List[JsValue]].size must equalTo(0) + + result = getEdges(query(12, direction = "out")) + println(result) + (result \ "results").as[List[JsValue]].size must equalTo(0) + + result = getEdges(query(10, direction = "out")) + println(result) + // 10 -> out -> 20 should not be in result. + (result \ "results").as[List[JsValue]].size must equalTo(1) + (result \\ "to").size must equalTo(1) + (result \\ "to").head.as[String] must equalTo("21") + + result = getEdges(query(20, direction = "in", columnName = testTgtColumnName)) + println(result) + (result \ "results").as[List[JsValue]].size must equalTo(0) + + val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(startTs = deletedAt + 1), withWait = true)) + + result = getEdges(query(20, direction = "in", columnName = testTgtColumnName)) + println(result) + (result \ "results").as[List[JsValue]].size must equalTo(3) + + true + + } + } + } + + + "labelargeSet of contention" should { + val labelName = testLabelName2 + val maxTgtId = 10 + val batchSize = 10 + val testNum = 3 + val numOfBatch = 10 + + def testInner(startTs: Long, src: Long) = { + val labelName = testLabelName2 + val lastOps = Array.fill(maxTgtId)("none") + var currentTs = startTs + + val allRequests = for { + ith <- (0 until numOfBatch) + jth <- (0 until batchSize) + } yield { + currentTs += 1 + + val tgt = Random.nextInt(maxTgtId) + val op = if (Random.nextDouble() < 0.5) "delete" else "update" + + lastOps(tgt) = op + Seq(currentTs, op, "e", src, src + tgt, labelName, "{}").mkString("\t") + } + + + allRequests.foreach(println(_)) +// println(lastOps.count(op => op != "delete" && op != "none")) +// println(lastOps) +// +// Thread.sleep(1000) + + val futures = Random.shuffle(allRequests).grouped(batchSize).map { bulkRequest => + val bulkEdge = bulkRequest.mkString("\n") + EdgeController.mutateAndPublish(bulkEdge, withWait = true) + } + + Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) + + val expectedDegree = lastOps.count(op => op != "delete" && op != "none") + val queryJson = query(id = src) + val result = getEdges(queryJson) + val resultSize = (result \ "size").as[Long] + val resultDegree = getDegree(result) + + println(lastOps.toList) + println(result) + + val ret = resultDegree == expectedDegree && resultSize == resultDegree + if (!ret) System.err.println(s"[Contention Failed]: $resultDegree, $expectedDegree") + + (ret, currentTs) + } + + "update delete" in { + running(FakeApplication()) { + val ret = for { + i <- (0 until testNum) + } yield { + val src = System.currentTimeMillis() + + val (ret, last) = testInner(i, src) + ret must beEqualTo(true) + ret + } + + ret.forall(identity) + } + } + + "update delete 2" in { + running(FakeApplication()) { + + val src = System.currentTimeMillis() + var ts = 0L + + val ret = for { + i <- (0 until testNum) + } yield { + val (ret, lastTs) = testInner(ts, src) + val deletedAt = lastTs + 1 + val deletedAt2 = lastTs + 2 + ts = deletedAt2 + 1 // nex start ts + + ret must beEqualTo(true) + + + val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt)) + val deleteAllRequest2 = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt2)) + + val deleteRet = EdgeController.deleteAllInner(deleteAllRequest, withWait = true) + val deleteRet2 = EdgeController.deleteAllInner(deleteAllRequest2, withWait = true) + + + Await.result(deleteRet, Duration(20, TimeUnit.MINUTES)) + Await.result(deleteRet2, Duration(20, TimeUnit.MINUTES)) + + val result = getEdges(query(id = src)) + println(result) + + val resultEdges = (result \ "results").as[Seq[JsValue]] + resultEdges.isEmpty must beEqualTo(true) + + val degreeAfterDeleteAll = getDegree(result) + degreeAfterDeleteAll must beEqualTo(0) + true + } + + ret.forall(identity) + } + } + + /** this test stress out test on degree + * when contention is low but number of adjacent edges are large */ + "large degrees" in { + running(FakeApplication()) { + + + val labelName = testLabelName2 + val dir = "out" + val maxSize = 100 + val deleteSize = 10 + val numOfConcurrentBatch = 100 + val src = System.currentTimeMillis() + val tgts = (0 until maxSize).map { ith => src + ith } + val deleteTgts = Random.shuffle(tgts).take(deleteSize) + val insertRequests = tgts.map { tgt => + Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t") + } + val deleteRequests = deleteTgts.take(deleteSize).map { tgt => + Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t") + } + val allRequests = Random.shuffle(insertRequests ++ deleteRequests) + // val allRequests = insertRequests ++ deleteRequests + val futures = allRequests.grouped(numOfConcurrentBatch).map { requests => + EdgeController.mutateAndPublish(requests.mkString("\n"), withWait = true) + } + Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) + + val expectedDegree = insertRequests.size - deleteRequests.size + val queryJson = query(id = src) + val result = getEdges(queryJson) + val resultSize = (result \ "size").as[Long] + val resultDegree = getDegree(result) + + // println(result) + + val ret = resultSize == expectedDegree && resultDegree == resultSize + println(s"[MaxSize]: $maxSize") + println(s"[DeleteSize]: $deleteSize") + println(s"[ResultDegree]: $resultDegree") + println(s"[ExpectedDegree]: $expectedDegree") + println(s"[ResultSize]: $resultSize") + ret must beEqualTo(true) + } + } + + "deleteAll" in { + running(FakeApplication()) { + + val labelName = testLabelName2 + val dir = "out" + val maxSize = 100 + val deleteSize = 10 + val numOfConcurrentBatch = 100 + val src = System.currentTimeMillis() + val tgts = (0 until maxSize).map { ith => src + ith } + val deleteTgts = Random.shuffle(tgts).take(deleteSize) + val insertRequests = tgts.map { tgt => + Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t") + } + val deleteRequests = deleteTgts.take(deleteSize).map { tgt => + Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t") + } + val allRequests = Random.shuffle(insertRequests ++ deleteRequests) + // val allRequests = insertRequests ++ deleteRequests + val futures = allRequests.grouped(numOfConcurrentBatch).map { requests => + EdgeController.mutateAndPublish(requests.mkString("\n"), withWait = true) + } + Await.result(Future.sequence(futures), Duration(10, TimeUnit.MINUTES)) + + val deletedAt = System.currentTimeMillis() + val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt)) + + Await.result(EdgeController.deleteAllInner(deleteAllRequest, withWait = true), Duration(10, TimeUnit.MINUTES)) + + val result = getEdges(query(id = src)) + println(result) + val resultEdges = (result \ "results").as[Seq[JsValue]] + resultEdges.isEmpty must beEqualTo(true) + + val degreeAfterDeleteAll = getDegree(result) + degreeAfterDeleteAll must beEqualTo(0) + } + } + } +} + diff --git a/s2rest_play/test/controllers/VertexSpec.scala b/s2rest_play/test/controllers/VertexSpec.scala new file mode 100644 index 00000000..f427b578 --- /dev/null +++ b/s2rest_play/test/controllers/VertexSpec.scala @@ -0,0 +1,39 @@ +package controllers + +import play.api.libs.json._ +import play.api.test.FakeApplication +import play.api.test.Helpers._ + +class VertexSpec extends SpecCommon { + // init() + + "vetex tc" should { + "tc1" in { + + running(FakeApplication()) { + val ids = (7 until 20).map(tcNum => tcNum * 1000 + 0) + + val (serviceName, columnName) = (testServiceName, testColumnName) + + val data = vertexInsertsPayload(serviceName, columnName, ids) + val payload = Json.parse(Json.toJson(data).toString) + println(payload) + + val jsResult = contentAsString(VertexController.tryMutates(payload, "insert", + Option(serviceName), Option(columnName), withWait = true)) + + val query = vertexQueryJson(serviceName, columnName, ids) + val ret = contentAsJson(QueryController.getVerticesInner(query)) + println(">>>", ret) + val fetched = ret.as[Seq[JsValue]] + for { + (d, f) <- data.zip(fetched) + } yield { + (d \ "id") must beEqualTo((f \ "id")) + ((d \ "props") \ "age") must beEqualTo(((f \ "props") \ "age")) + } + } + true + } + } +} diff --git a/s2rest_play/test/controllers/WeakLabelDeleteSpec.scala b/s2rest_play/test/controllers/WeakLabelDeleteSpec.scala new file mode 100644 index 00000000..164c52bc --- /dev/null +++ b/s2rest_play/test/controllers/WeakLabelDeleteSpec.scala @@ -0,0 +1,126 @@ +package controllers + +import play.api.libs.json._ +import play.api.test.Helpers._ +import play.api.test.{FakeApplication, FakeRequest} + +class WeakLabelDeleteSpec extends SpecCommon { + init() + + def bulkEdges(startTs: Int = 0) = Seq( + Seq(startTs + 1, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"), + Seq(startTs + 2, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 11}""").mkString("\t"), + Seq(startTs + 3, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 12}""").mkString("\t"), + Seq(startTs + 4, "insert", "e", "0", "2", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"), + Seq(startTs + 5, "insert", "e", "10", "20", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"), + Seq(startTs + 6, "insert", "e", "10", "21", testLabelNameWeak, s"""{"time": 11}""").mkString("\t"), + Seq(startTs + 7, "insert", "e", "11", "20", testLabelNameWeak, s"""{"time": 12}""").mkString("\t"), + Seq(startTs + 8, "insert", "e", "12", "20", testLabelNameWeak, s"""{"time": 13}""").mkString("\t") + ).mkString("\n") + + "weak label delete test" should { + running(FakeApplication()) { + // insert bulk and wait .. + val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(), withWait = true)) + } + + + def query(id: Int, direction: String = "out", columnName: String = testColumnName) = Json.parse( + s""" + { "srcVertices": [ + { "serviceName": "$testServiceName", + "columnName": "$columnName", + "id": ${id} + }], + "steps": [ + [ { + "label": "${testLabelNameWeak}", + "direction": "${direction}", + "offset": 0, + "limit": 10, + "duplicate": "raw" + } + ]] + }""") + + def getEdges(queryJson: JsValue): JsValue = { + val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get + contentAsJson(ret) + } + + "test weak consistency select" in { + running(FakeApplication()) { + var result = getEdges(query(0)) + println(result) + (result \ "results").as[List[JsValue]].size must equalTo(4) + result = getEdges(query(10)) + println(result) + (result \ "results").as[List[JsValue]].size must equalTo(2) + true + } + } + + "test weak consistency delete" in { + running(FakeApplication()) { + var result = getEdges(query(0)) + println(result) + + /** expect 4 edges */ + (result \ "results").as[List[JsValue]].size must equalTo(4) + val edges = (result \ "results").as[List[JsObject]] + + contentAsJson(EdgeController.tryMutates(Json.toJson(edges), "delete", withWait = true)) + + /** expect noting */ + result = getEdges(query(0)) + println(result) + (result \ "results").as[List[JsValue]].size must equalTo(0) + + /** insert should be ignored */ + contentAsJson(EdgeController.tryMutates(Json.toJson(edges), "insert", withWait = true)) + + result = getEdges(query(0)) + (result \ "results").as[List[JsValue]].size must equalTo(0) + } + } + + "test weak consistency deleteAll" in { + running(FakeApplication()) { + val deletedAt = 100 + var result = getEdges(query(20, "in", testTgtColumnName)) + println(result) + (result \ "results").as[List[JsValue]].size must equalTo(3) + + val json = Json.arr(Json.obj("label" -> testLabelNameWeak, + "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt)) + println(json) + contentAsString(EdgeController.deleteAllInner(json, withWait = true)) + + + result = getEdges(query(11, "out")) + (result \ "results").as[List[JsValue]].size must equalTo(0) + + result = getEdges(query(12, "out")) + (result \ "results").as[List[JsValue]].size must equalTo(0) + + result = getEdges(query(10, "out")) + + // 10 -> out -> 20 should not be in result. + (result \ "results").as[List[JsValue]].size must equalTo(1) + (result \\ "to").size must equalTo(1) + (result \\ "to").head.as[String] must equalTo("21") + + result = getEdges(query(20, "in", testTgtColumnName)) + println(result) + (result \ "results").as[List[JsValue]].size must equalTo(0) + + val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(startTs = deletedAt + 1), withWait = true)) + + result = getEdges(query(20, "in", testTgtColumnName)) + (result \ "results").as[List[JsValue]].size must equalTo(3) + } + } + } + +} + From 32b1c4a8b38c22053ac2863d8948708d8bd03a0d Mon Sep 17 00:00:00 2001 From: daewon Date: Thu, 14 Jan 2016 18:13:02 +0900 Subject: [PATCH 05/13] S2GRAPH-7 Move Test from root to s2core --- .../scala/com/kakao/s2graph/core/Graph.scala | 2 +- .../s2graph/core/types/InnerValTest.scala | 1 - s2rest_play/conf/routes | 1 - .../benchmark/PostProcessBenchmarkSpec.scala | 241 ------- .../controllers/AdminControllerSpec.scala | 27 - .../test/controllers/BasicCrudSpec.scala | 251 -------- .../test/controllers/EdgeControllerSpec.scala | 22 - .../test/controllers/QueryCacheSpec.scala | 89 --- s2rest_play/test/controllers/QuerySpec.scala | 604 ------------------ s2rest_play/test/controllers/SpecCommon.scala | 365 ----------- .../controllers/StrongLabelDeleteSpec.scala | 345 ---------- s2rest_play/test/controllers/VertexSpec.scala | 39 -- .../controllers/WeakLabelDeleteSpec.scala | 126 ---- 13 files changed, 1 insertion(+), 2112 deletions(-) delete mode 100644 s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala delete mode 100644 s2rest_play/test/controllers/AdminControllerSpec.scala delete mode 100644 s2rest_play/test/controllers/BasicCrudSpec.scala delete mode 100644 s2rest_play/test/controllers/EdgeControllerSpec.scala delete mode 100644 s2rest_play/test/controllers/QueryCacheSpec.scala delete mode 100644 s2rest_play/test/controllers/QuerySpec.scala delete mode 100644 s2rest_play/test/controllers/SpecCommon.scala delete mode 100644 s2rest_play/test/controllers/StrongLabelDeleteSpec.scala delete mode 100644 s2rest_play/test/controllers/VertexSpec.scala delete mode 100644 s2rest_play/test/controllers/WeakLabelDeleteSpec.scala diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala b/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala index 111f3df1..5fc44ccd 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala @@ -312,7 +312,7 @@ object Graph { } get } -class Graph(_config: Config)(implicit ec: ExecutionContext) { +class Graph(_config: Config)(implicit val ec: ExecutionContext) { val config = _config.withFallback(Graph.DefaultConfig) val cacheSize = config.getInt("cache.max.size") // val cache = CacheBuilder.newBuilder().maximumSize(cacheSize).build[java.lang.Integer, Seq[QueryResult]]() diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala index 69a299de..9e01da3f 100644 --- a/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala +++ b/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala @@ -8,7 +8,6 @@ import play.api.libs.json.Json class InnerValTest extends FunSuite with Matchers with TestCommonWithModels { initTests() - import HBaseType.{VERSION2, VERSION1} val decimals = List( BigDecimal(Long.MinValue), diff --git a/s2rest_play/conf/routes b/s2rest_play/conf/routes index 5282b7b0..df4a1eed 100644 --- a/s2rest_play/conf/routes +++ b/s2rest_play/conf/routes @@ -25,7 +25,6 @@ POST /graphs/edges/updateWithWait contro POST /graphs/edges/increment controllers.EdgeController.increments() POST /graphs/edges/incrementCount controllers.EdgeController.incrementCounts() POST /graphs/edges/bulk controllers.EdgeController.mutateBulk() -POST /graphs/edges/bulkWithWait controllers.EdgeController.mutateBulkWithWait() ## Vertex POST /graphs/vertices/insert controllers.VertexController.inserts() diff --git a/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala b/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala deleted file mode 100644 index 22a7b5a1..00000000 --- a/s2rest_play/test/benchmark/PostProcessBenchmarkSpec.scala +++ /dev/null @@ -1,241 +0,0 @@ -package benchmark - -import com.kakao.s2graph.core.mysqls.Label -import com.kakao.s2graph.core.rest.RequestParser -import com.kakao.s2graph.core.{PostProcess, Graph, Management} -import com.typesafe.config.ConfigFactory -import controllers._ -import play.api.libs.json.{JsValue, Json} -import play.api.test.{FakeApplication, FakeRequest, PlaySpecification} - -import scala.concurrent.Await -import scala.concurrent.duration._ - -/** - * Created by hsleep(honeysleep@gmail.com) on 2015. 11. 6.. - */ -class PostProcessBenchmarkSpec extends SpecCommon with BenchmarkCommon with PlaySpecification { - sequential - - import Helper._ - - init() - - override def init() = { - running(FakeApplication()) { - println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") - Management.deleteService(testServiceName) - - // 1. createService - val result = AdminController.createServiceInner(Json.parse(createService)) - println(s">> Service created : $createService, $result") - - val labelNames = Map( - testLabelNameWeak -> testLabelNameWeakCreate - ) - - for { - (labelName, create) <- labelNames - } { - Management.deleteLabel(labelName) - Label.findByName(labelName, useCache = false) match { - case None => - AdminController.createLabelInner(Json.parse(create)) - case Some(label) => - println(s">> Label already exist: $create, $label") - } - } - - // create edges - val bulkEdges: String = (0 until 500).map { i => - edge"${System.currentTimeMillis()} insert e 0 $i $testLabelNameWeak"($(weight=i)) - }.mkString("\n") - - val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) - - println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") - } - } - - def getEdges(queryJson: JsValue): JsValue = { - val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get - contentAsJson(ret) - } - - val s2: Graph = com.kakao.s2graph.rest.Global.s2graph - -// "test performance of getEdges orderBy" >> { -// running(FakeApplication()) { -// val strJs = -// s""" -// |{ -// | "orderBy": [ -// | {"score": "DESC"}, -// | {"timestamp": "DESC"} -// | ], -// | "srcVertices": [ -// | { -// | "serviceName": "$testServiceName", -// | "columnName": "$testColumnName", -// | "ids": [0] -// | } -// | ], -// | "steps": [ -// | { -// | "step": [ -// | { -// | "cacheTTL": 60000, -// | "label": "$testLabelNameWeak", -// | "offset": 0, -// | "limit": -1, -// | "direction": "out", -// | "scoring": [ -// | {"weight": 1} -// | ] -// | } -// | ] -// | } -// | ] -// |} -// """.stripMargin -// -// object Parser extends RequestParser -// -// val js = Json.parse(strJs) -// -// val q = Parser.toQuery(js) -// -// val queryResultLs = Await.result(s2.getEdges(q), 1 seconds) -// -// val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs) -// -// (resultJs \ "size").as[Int] must_== 500 -// -// (0 to 5) foreach { _ => -// duration("toSimpleVertexArrJson new orderBy") { -// (0 to 1000) foreach { _ => -// PostProcess.toSimpleVertexArrJson(queryResultLs, Nil) -// } -// } -// } -// -// (resultJs \ "size").as[Int] must_== 500 -// } -// } - - "test performance of getEdges" >> { - running(FakeApplication()) { - val strJs = - s""" - |{ - | "srcVertices": [ - | { - | "serviceName": "$testServiceName", - | "columnName": "$testColumnName", - | "ids": [0] - | } - | ], - | "steps": [ - | { - | "step": [ - | { - | "cacheTTL": 60000, - | "label": "$testLabelNameWeak", - | "offset": 0, - | "limit": -1, - | "direction": "out", - | "scoring": [ - | {"weight": 1} - | ] - | } - | ] - | } - | ] - |} - """.stripMargin - - val config = ConfigFactory.load() - val requestParser = new RequestParser(config) - - val js = Json.parse(strJs) - - val q = requestParser.toQuery(js) - - val queryResultLs = Await.result(s2.getEdges(q), 1 seconds) - - val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs, Nil) - - (0 to 5) foreach { _ => - duration("toSimpleVertexArrJson new") { - (0 to 1000) foreach { _ => - PostProcess.toSimpleVertexArrJson(queryResultLs, Nil) - } - } - } - - (resultJs \ "size").as[Int] must_== 500 - } - } - -// "test performance of getEdges withScore=false" >> { -// running(FakeApplication()) { -// val strJs = -// s""" -// |{ -// | "withScore": false, -// | "srcVertices": [ -// | { -// | "serviceName": "$testServiceName", -// | "columnName": "$testColumnName", -// | "ids": [0] -// | } -// | ], -// | "steps": [ -// | { -// | "step": [ -// | { -// | "cacheTTL": 60000, -// | "label": "$testLabelNameWeak", -// | "offset": 0, -// | "limit": -1, -// | "direction": "out", -// | "scoring": [ -// | {"weight": 1} -// | ] -// | } -// | ] -// | } -// | ] -// |} -// """.stripMargin -// -// object Parser extends RequestParser -// -// val js = Json.parse(strJs) -// -// val q = Parser.toQuery(js) -// -// val queryResultLs = Await.result(s2.getEdges(q), 1 seconds) -// -// val resultJs = PostProcess.toSimpleVertexArrJson(queryResultLs) -// -// (resultJs \ "size").as[Int] must_== 500 -// -// (0 to 5) foreach { _ => -// duration("toSimpleVertexArrJson withScore=false org") { -// (0 to 1000) foreach { _ => -// PostProcess.toSimpleVertexArrJsonOrg(queryResultLs, Nil) -// } -// } -// -// duration("toSimpleVertexArrJson withScore=false new") { -// (0 to 1000) foreach { _ => -// PostProcess.toSimpleVertexArrJson(queryResultLs, Nil) -// } -// } -// } -// -// (resultJs \ "size").as[Int] must_== 500 -// } -// } -} diff --git a/s2rest_play/test/controllers/AdminControllerSpec.scala b/s2rest_play/test/controllers/AdminControllerSpec.scala deleted file mode 100644 index e41fccb0..00000000 --- a/s2rest_play/test/controllers/AdminControllerSpec.scala +++ /dev/null @@ -1,27 +0,0 @@ -package controllers - -import com.kakao.s2graph.core.mysqls.Label -import play.api.http.HeaderNames -import play.api.test.Helpers._ -import play.api.test.{FakeApplication, FakeRequest} - -import scala.concurrent.Await - -/** - * Created by mojo22jojo(hyunsung.jo@gmail.com) on 15. 10. 13.. - */ -class AdminControllerSpec extends SpecCommon { - init() - "EdgeControllerSpec" should { - "update htable" in { - running(FakeApplication()) { - val insertUrl = s"/graphs/updateHTable/$testLabelName/$newHTableName" - - val req = FakeRequest("POST", insertUrl).withBody("").withHeaders(HeaderNames.CONTENT_TYPE -> "text/plain") - - Await.result(route(req).get, HTTP_REQ_WAITING_TIME) - Label.findByName(testLabelName, useCache = true).get.hTableName mustEqual newHTableName - } - } - } -} diff --git a/s2rest_play/test/controllers/BasicCrudSpec.scala b/s2rest_play/test/controllers/BasicCrudSpec.scala deleted file mode 100644 index f4f11b45..00000000 --- a/s2rest_play/test/controllers/BasicCrudSpec.scala +++ /dev/null @@ -1,251 +0,0 @@ -package controllers - -import com.kakao.s2graph.core.Management -import com.kakao.s2graph.core.mysqls._ - -//import com.kakao.s2graph.core.models._ - -import play.api.libs.json._ -import play.api.test.Helpers._ -import play.api.test.{FakeApplication, FakeRequest} - -import scala.concurrent.Await - - -class BasicCrudSpec extends SpecCommon { - sequential - - var seed = 0 - def runTC(tcNum: Int, tcString: String, opWithProps: List[(Long, String, String)], expected: Map[String, String]) = { - for { - labelName <- List(testLabelName, testLabelName2) - i <- 0 until NUM_OF_EACH_TEST - } { - seed += 1 -// val srcId = ((tcNum * 1000) + i).toString -// val tgtId = if (labelName == testLabelName) s"${srcId + 1000 + i}" else s"${srcId + 1000 + i}abc" - val srcId = seed.toString - val tgtId = srcId - - val maxTs = opWithProps.map(t => t._1).max - - /** insert edges */ - println(s"---- TC${tcNum}_init ----") - val bulkEdge = (for ((ts, op, props) <- opWithProps) yield { - List(ts, op, "e", srcId, tgtId, labelName, props).mkString("\t") - }).mkString("\n") - - val req = EdgeController.mutateAndPublish(bulkEdge, withWait = true) - val res = Await.result(req, HTTP_REQ_WAITING_TIME) - - res.header.status must equalTo(200) - - println(s"---- TC${tcNum}_init ----") -// Thread.sleep(100) - - for { - label <- Label.findByName(labelName) - direction <- List("out", "in") - cacheTTL <- List(-1L) - } { - val (serviceName, columnName, id, otherId) = direction match { - case "out" => (label.srcService.serviceName, label.srcColumn.columnName, srcId, tgtId) - case "in" => (label.tgtService.serviceName, label.tgtColumn.columnName, tgtId, srcId) - } - val qId = if (labelName == testLabelName) id else "\"" + id + "\"" - val query = queryJson(serviceName, columnName, labelName, qId, direction, cacheTTL) - val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(query)).get - val jsResult = commonCheck(ret) - - val results = jsResult \ "results" - val deegrees = (jsResult \ "degrees").as[List[JsObject]] - val propsLs = (results \\ "props").seq - (deegrees.head \ LabelMeta.degree.name).as[Int] must equalTo(1) - - val from = (results \\ "from").seq.last.toString.replaceAll("\"", "") - val to = (results \\ "to").seq.last.toString.replaceAll("\"", "") - - from must equalTo(id.toString) - to must equalTo(otherId.toString) -// (results \\ "_timestamp").seq.last.as[Long] must equalTo(maxTs) - for ((key, expectedVal) <- expected) { - propsLs.last.as[JsObject].keys.contains(key) must equalTo(true) - (propsLs.last \ key).toString must equalTo(expectedVal) - } - Await.result(ret, HTTP_REQ_WAITING_TIME) - } - } - } - - init() - "Basic Crud " should { - "tc1" in { - running(FakeApplication()) { - - var tcNum = 0 - var tcString = "" - var bulkQueries = List.empty[(Long, String, String)] - var expected = Map.empty[String, String] - - tcNum = 7 - tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test " - bulkQueries = List( - (t1, "insert", "{\"time\": 10}"), - (t2, "delete", ""), - (t3, "insert", "{\"time\": 10, \"weight\": 20}")) - expected = Map("time" -> "10", "weight" -> "20") - - runTC(tcNum, tcString, bulkQueries, expected) - - tcNum = 8 - tcString = "[t1 -> t2 -> t3 test case] insert(t1) delete(t2) insert(t3) test " - bulkQueries = List( - (t1, "insert", "{\"time\": 10}"), - (t3, "insert", "{\"time\": 10, \"weight\": 20}"), - (t2, "delete", "")) - expected = Map("time" -> "10", "weight" -> "20") - - runTC(tcNum, tcString, bulkQueries, expected) - - tcNum = 9 - tcString = "[t3 -> t2 -> t1 test case] insert(t3) delete(t2) insert(t1) test " - bulkQueries = List( - (t3, "insert", "{\"time\": 10, \"weight\": 20}"), - (t2, "delete", ""), - (t1, "insert", "{\"time\": 10}")) - expected = Map("time" -> "10", "weight" -> "20") - - runTC(tcNum, tcString, bulkQueries, expected) - - tcNum = 10 - tcString = "[t3 -> t1 -> t2 test case] insert(t3) insert(t1) delete(t2) test " - bulkQueries = List( - (t3, "insert", "{\"time\": 10, \"weight\": 20}"), - (t1, "insert", "{\"time\": 10}"), - (t2, "delete", "")) - expected = Map("time" -> "10", "weight" -> "20") - - runTC(tcNum, tcString, bulkQueries, expected) - - tcNum = 11 - tcString = "[t2 -> t1 -> t3 test case] delete(t2) insert(t1) insert(t3) test" - bulkQueries = List( - (t2, "delete", ""), - (t1, "insert", "{\"time\": 10}"), - (t3, "insert", "{\"time\": 10, \"weight\": 20}")) - expected = Map("time" -> "10", "weight" -> "20") - - runTC(tcNum, tcString, bulkQueries, expected) - - tcNum = 12 - tcString = "[t2 -> t3 -> t1 test case] delete(t2) insert(t3) insert(t1) test " - bulkQueries = List( - (t2, "delete", ""), - (t3, "insert", "{\"time\": 10, \"weight\": 20}"), - (t1, "insert", "{\"time\": 10}")) - expected = Map("time" -> "10", "weight" -> "20") - - runTC(tcNum, tcString, bulkQueries, expected) - - tcNum = 13 - tcString = "[t1 -> t2 -> t3 test case] update(t1) delete(t2) update(t3) test " - bulkQueries = List( - (t1, "update", "{\"time\": 10}"), - (t2, "delete", ""), - (t3, "update", "{\"time\": 10, \"weight\": 20}")) - expected = Map("time" -> "10", "weight" -> "20") - - runTC(tcNum, tcString, bulkQueries, expected) - tcNum = 14 - tcString = "[t1 -> t3 -> t2 test case] update(t1) update(t3) delete(t2) test " - bulkQueries = List( - (t1, "update", "{\"time\": 10}"), - (t3, "update", "{\"time\": 10, \"weight\": 20}"), - (t2, "delete", "")) - expected = Map("time" -> "10", "weight" -> "20") - - runTC(tcNum, tcString, bulkQueries, expected) - tcNum = 15 - tcString = "[t2 -> t1 -> t3 test case] delete(t2) update(t1) update(t3) test " - bulkQueries = List( - (t2, "delete", ""), - (t1, "update", "{\"time\": 10}"), - (t3, "update", "{\"time\": 10, \"weight\": 20}")) - expected = Map("time" -> "10", "weight" -> "20") - - runTC(tcNum, tcString, bulkQueries, expected) - tcNum = 16 - tcString = "[t2 -> t3 -> t1 test case] delete(t2) update(t3) update(t1) test" - bulkQueries = List( - (t2, "delete", ""), - (t3, "update", "{\"time\": 10, \"weight\": 20}"), - (t1, "update", "{\"time\": 10}")) - expected = Map("time" -> "10", "weight" -> "20") - - runTC(tcNum, tcString, bulkQueries, expected) - tcNum = 17 - tcString = "[t3 -> t2 -> t1 test case] update(t3) delete(t2) update(t1) test " - bulkQueries = List( - (t3, "update", "{\"time\": 10, \"weight\": 20}"), - (t2, "delete", ""), - (t1, "update", "{\"time\": 10}")) - expected = Map("time" -> "10", "weight" -> "20") - - runTC(tcNum, tcString, bulkQueries, expected) - tcNum = 18 - tcString = "[t3 -> t1 -> t2 test case] update(t3) update(t1) delete(t2) test " - bulkQueries = List( - (t3, "update", "{\"time\": 10, \"weight\": 20}"), - (t1, "update", "{\"time\": 10}"), - (t2, "delete", "")) - expected = Map("time" -> "10", "weight" -> "20") - - runTC(tcNum, tcString, bulkQueries, expected) - - tcNum = 19 - tcString = "[t5 -> t1 -> t3 -> t2 -> t4 test case] update(t5) insert(t1) insert(t3) delete(t2) update(t4) test " - bulkQueries = List( - (t5, "update", "{\"is_blocked\": true}"), - (t1, "insert", "{\"is_hidden\": false}"), - (t3, "insert", "{\"is_hidden\": false, \"weight\": 10}"), - (t2, "delete", ""), - (t4, "update", "{\"time\": 1, \"weight\": -10}")) - expected = Map("time" -> "1", "weight" -> "-10", "is_hidden" -> "false", "is_blocked" -> "true") - - runTC(tcNum, tcString, bulkQueries, expected) - true - } - } - } - - "toLogString" in { - running(FakeApplication()) { - val bulkQueries = List( - ("1445240543366", "update", "{\"is_blocked\":true}"), - ("1445240543362", "insert", "{\"is_hidden\":false}"), - ("1445240543364", "insert", "{\"is_hidden\":false,\"weight\":10}"), - ("1445240543363", "delete", "{}"), - ("1445240543365", "update", "{\"time\":1, \"weight\":-10}")) - - val (srcId, tgtId, labelName) = ("1", "2", testLabelName) - - val bulkEdge = (for ((ts, op, props) <- bulkQueries) yield { - Management.toEdge(ts.toLong, op, srcId, tgtId, labelName, "out", props).toLogString - }).mkString("\n") - - val expected = Seq( - Seq("1445240543366", "update", "e", "1", "2", "s2graph_label_test", "{\"is_blocked\":true}"), - Seq("1445240543362", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false}"), - Seq("1445240543364", "insert", "e", "1", "2", "s2graph_label_test", "{\"is_hidden\":false,\"weight\":10}"), - Seq("1445240543363", "delete", "e", "1", "2", "s2graph_label_test"), - Seq("1445240543365", "update", "e", "1", "2", "s2graph_label_test", "{\"time\":1,\"weight\":-10}") - ).map(_.mkString("\t")).mkString("\n") - - bulkEdge must equalTo(expected) - - true - } - } -} - - diff --git a/s2rest_play/test/controllers/EdgeControllerSpec.scala b/s2rest_play/test/controllers/EdgeControllerSpec.scala deleted file mode 100644 index e76404ab..00000000 --- a/s2rest_play/test/controllers/EdgeControllerSpec.scala +++ /dev/null @@ -1,22 +0,0 @@ -package controllers - -import play.api.http.HeaderNames -import play.api.test.{FakeApplication, FakeRequest, PlaySpecification, WithApplication} -import play.api.{Application => PlayApplication} - -/** - * Created by hsleep(honeysleep@gmail.com) on 15. 9. 1.. - */ -class EdgeControllerSpec extends PlaySpecification { -// "EdgeControllerSpec" should { -// implicit val app = FakeApplication() -// -// "bad request invalid json" in new WithApplication(app) { -// val insertUrl = "http://localhost:9000/graphs/edges/insert" -// val req = FakeRequest("POST", insertUrl).withBody("").withHeaders(HeaderNames.CONTENT_TYPE -> "application/json") -// val result = EdgeController.inserts().apply(req).run -// -// status(result) must_== BAD_REQUEST -// } -// } -} diff --git a/s2rest_play/test/controllers/QueryCacheSpec.scala b/s2rest_play/test/controllers/QueryCacheSpec.scala deleted file mode 100644 index 7d91aa98..00000000 --- a/s2rest_play/test/controllers/QueryCacheSpec.scala +++ /dev/null @@ -1,89 +0,0 @@ -//package test.controllers -// -////import com.kakao.s2graph.core.models._ -// -//import controllers.EdgeController -//import play.api.libs.json._ -//import play.api.test.Helpers._ -//import play.api.test.{FakeApplication, FakeRequest} -// -//class QueryCacheSpec extends SpecCommon { -// init() -// -// "cache test" should { -// def queryWithTTL(id: Int, cacheTTL: Long) = Json.parse( s""" -// { "srcVertices": [ -// { "serviceName": "${testServiceName}", -// "columnName": "${testColumnName}", -// "id": ${id} -// }], -// "steps": [[ { -// "label": "${testLabelName}", -// "direction": "out", -// "offset": 0, -// "limit": 10, -// "cacheTTL": ${cacheTTL}, -// "scoring": {"weight": 1} }]] -// }""") -// -// def getEdges(queryJson: JsValue): JsValue = { -// var ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get -// contentAsJson(ret) -// } -// -// // init -// running(FakeApplication()) { -// // insert bulk and wait .. -// val bulkEdges: String = Seq( -// Seq("1", "insert", "e", "0", "2", "s2graph_label_test", "{}").mkString("\t"), -// Seq("1", "insert", "e", "1", "2", "s2graph_label_test", "{}").mkString("\t") -// ).mkString("\n") -// -// val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) -// Thread.sleep(asyncFlushInterval) -// } -// -// "tc1: query with {id: 0, ttl: 1000}" in { -// running(FakeApplication()) { -// var jsRslt = getEdges(queryWithTTL(0, 1000)) -// var cacheRemain = (jsRslt \\ "cacheRemain").head -// cacheRemain.as[Int] must greaterThan(500) -// -// // get edges from cache after wait 500ms -// Thread.sleep(500) -// val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryWithTTL(0, 1000))).get -// jsRslt = contentAsJson(ret) -// cacheRemain = (jsRslt \\ "cacheRemain").head -// cacheRemain.as[Int] must lessThan(500) -// } -// } -// -// "tc2: query with {id: 1, ttl: 3000}" in { -// running(FakeApplication()) { -// var jsRslt = getEdges(queryWithTTL(1, 3000)) -// var cacheRemain = (jsRslt \\ "cacheRemain").head -// // before update: is_blocked is false -// (jsRslt \\ "is_blocked").head must equalTo(JsBoolean(false)) -// -// val bulkEdges = Seq( -// Seq("2", "update", "e", "0", "2", "s2graph_label_test", "{\"is_blocked\": true}").mkString("\t"), -// Seq("2", "update", "e", "1", "2", "s2graph_label_test", "{\"is_blocked\": true}").mkString("\t") -// ).mkString("\n") -// -// // update edges with {is_blocked: true} -// jsRslt = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) -// -// Thread.sleep(asyncFlushInterval) -// -// // prop 'is_blocked' still false, cause queryResult on cache -// jsRslt = getEdges(queryWithTTL(1, 3000)) -// (jsRslt \\ "is_blocked").head must equalTo(JsBoolean(false)) -// -// // after wait 3000ms prop 'is_blocked' is updated to true, cache cleared -// Thread.sleep(3000) -// jsRslt = getEdges(queryWithTTL(1, 3000)) -// (jsRslt \\ "is_blocked").head must equalTo(JsBoolean(true)) -// } -// } -// } -//} diff --git a/s2rest_play/test/controllers/QuerySpec.scala b/s2rest_play/test/controllers/QuerySpec.scala deleted file mode 100644 index f6381bc0..00000000 --- a/s2rest_play/test/controllers/QuerySpec.scala +++ /dev/null @@ -1,604 +0,0 @@ -package controllers - -import play.api.libs.json._ -import play.api.test.{FakeApplication, FakeRequest, PlaySpecification} -import play.api.{Application => PlayApplication} - -import scala.concurrent.Await - -class QuerySpec extends SpecCommon with PlaySpecification { - - import Helper._ - - implicit val app = FakeApplication() - - init() - - "query test" should { - running(FakeApplication()) { - - // insert bulk and wait .. - val bulkEdges: String = Seq( - edge"1000 insert e 0 1 $testLabelName"($(weight = 40, is_hidden = true)), - edge"2000 insert e 0 2 $testLabelName"($(weight = 30, is_hidden = false)), - edge"3000 insert e 2 0 $testLabelName"($(weight = 20)), - edge"4000 insert e 2 1 $testLabelName"($(weight = 10)), - edge"3000 insert e 10 20 $testLabelName"($(weight = 20)), - edge"4000 insert e 20 20 $testLabelName"($(weight = 10)), - edge"1 insert e -1 1000 $testLabelName", - edge"1 insert e -1 2000 $testLabelName", - edge"1 insert e -1 3000 $testLabelName", - edge"1 insert e 1000 10000 $testLabelName", - edge"1 insert e 1000 11000 $testLabelName", - edge"1 insert e 2000 11000 $testLabelName", - edge"1 insert e 2000 12000 $testLabelName", - edge"1 insert e 3000 12000 $testLabelName", - edge"1 insert e 3000 13000 $testLabelName", - edge"1 insert e 10000 100000 $testLabelName", - edge"2 insert e 11000 200000 $testLabelName", - edge"3 insert e 12000 300000 $testLabelName").mkString("\n") - - val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) - } - - def queryParents(id: Long) = Json.parse( s""" - { - "returnTree": true, - "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - [ { - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "limit": 2 - } - ],[{ - "label": "${testLabelName}", - "direction": "in", - "offset": 0, - "limit": -1 - } - ]] - }""".stripMargin) - - def queryExclude(id: Int) = Json.parse( s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - [ { - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "limit": 2 - }, - { - "label": "${testLabelName}", - "direction": "in", - "offset": 0, - "limit": 2, - "exclude": true - } - ]] - }""") - - def queryTransform(id: Int, transforms: String) = Json.parse( s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - [ { - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "transform": $transforms - } - ]] - }""") - - def queryWhere(id: Int, where: String) = Json.parse( s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - [ { - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "limit": 100, - "where": "${where}" - } - ]] - }""") - - def querySingleWithTo(id: Int, offset: Int = 0, limit: Int = 100, to: Int) = Json.parse( s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - [ { - "label": "${testLabelName}", - "direction": "out", - "offset": $offset, - "limit": $limit, - "_to": $to - } - ]] - } - """) - - def querySingle(id: Int, offset: Int = 0, limit: Int = 100) = Json.parse( s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - [ { - "label": "${testLabelName}", - "direction": "out", - "offset": $offset, - "limit": $limit - } - ]] - } - """) - - def queryWithSampling(id: Int, sample: Int) = Json.parse( s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - { - "step": [{ - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "limit": 100, - "sample": ${sample} - }] - } - ] - }""") - - - def twoStepQueryWithSampling(id: Int, sample: Int) = Json.parse( s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - { - "step": [{ - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "limit": 100, - "sample": ${sample} - }] - }, - { - "step": [{ - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "limit": 100, - "sample": ${sample} - }] - } - ] - }""") - - def twoQueryWithSampling(id: Int, sample: Int) = Json.parse( s""" - { "srcVertices": [ - { "serviceName": "${testServiceName}", - "columnName": "${testColumnName}", - "id": ${id} - }], - "steps": [ - { - "step": [{ - "label": "${testLabelName}", - "direction": "out", - "offset": 0, - "limit": 50, - "sample": ${sample} - }, - { - "label": "${testLabelName2}", - "direction": "out", - "offset": 0, - "limit": 50 - }] - } - ] - }""") - - def queryUnion(id: Int, size: Int) = JsArray(List.tabulate(size)(_ => querySingle(id))) - - def queryGroupBy(id: Int, props: Seq[String]): JsValue = { - Json.obj( - "groupBy" -> props, - "srcVertices" -> Json.arr( - Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) - ), - "steps" -> Json.arr( - Json.obj( - "step" -> Json.arr( - Json.obj( - "label" -> testLabelName - ) - ) - ) - ) - ) - } - - def queryScore(id: Int, scoring: Map[String, Int]): JsValue = { - val q = Json.obj( - "srcVertices" -> Json.arr( - Json.obj( - "serviceName" -> testServiceName, - "columnName" -> testColumnName, - "id" -> id - ) - ), - "steps" -> Json.arr( - Json.obj( - "step" -> Json.arr( - Json.obj( - "label" -> testLabelName, - "scoring" -> scoring - ) - ) - ) - ) - ) - println(q) - q - } - - def queryOrderBy(id: Int, scoring: Map[String, Int], props: Seq[Map[String, String]]): JsValue = { - Json.obj( - "orderBy" -> props, - "srcVertices" -> Json.arr( - Json.obj("serviceName" -> testServiceName, "columnName" -> testColumnName, "id" -> id) - ), - "steps" -> Json.arr( - Json.obj( - "step" -> Json.arr( - Json.obj( - "label" -> testLabelName, - "scoring" -> scoring - ) - ) - ) - ) - ) - } - - def getEdges(queryJson: JsValue): JsValue = { - val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get - contentAsJson(ret) - } - - def queryIndex(ids: Seq[Int], indexName: String) = { - val $from = $a( - $(serviceName = testServiceName, - columnName = testColumnName, - ids = ids)) - - val $step = $a($(label = testLabelName, index = indexName)) - val $steps = $a($(step = $step)) - - val js = $(withScore = false, srcVertices = $from, steps = $steps).toJson - js - } - - def queryDuration(ids: Seq[Int], from: Int, to: Int) = { - val $from = $a( - $(serviceName = testServiceName, - columnName = testColumnName, - ids = ids)) - - val $step = $a($( - label = testLabelName, direction = "out", offset = 0, limit = 100, - duration = $(from = from, to = to))) - - val $steps = $a($(step = $step)) - - $(srcVertices = $from, steps = $steps).toJson - } - - "union query" in { - running(FakeApplication()) { - var result = getEdges(queryUnion(0, 2)) - result.as[List[JsValue]].size must equalTo(2) - - result = getEdges(queryUnion(0, 3)) - result.as[List[JsValue]].size must equalTo(3) - - result = getEdges(queryUnion(0, 4)) - result.as[List[JsValue]].size must equalTo(4) - - result = getEdges(queryUnion(0, 5)) - result.as[List[JsValue]].size must equalTo(5) - - val union = result.as[List[JsValue]].head - val single = getEdges(querySingle(0)) - - (union \\ "from").map(_.toString).sorted must equalTo((single \\ "from").map(_.toString).sorted) - (union \\ "to").map(_.toString).sorted must equalTo((single \\ "to").map(_.toString).sorted) - (union \\ "weight").map(_.toString).sorted must equalTo((single \\ "weight").map(_.toString).sorted) - } - } - - "get edge with where condition" in { - running(FakeApplication()) { - var result = getEdges(queryWhere(0, "is_hidden=false and _from in (-1, 0)")) - (result \ "results").as[List[JsValue]].size must equalTo(1) - - result = getEdges(queryWhere(0, "is_hidden=true and _to in (1)")) - (result \ "results").as[List[JsValue]].size must equalTo(1) - - result = getEdges(queryWhere(0, "_from=0")) - (result \ "results").as[List[JsValue]].size must equalTo(2) - - result = getEdges(queryWhere(2, "_from=2 or weight in (-1)")) - (result \ "results").as[List[JsValue]].size must equalTo(2) - - result = getEdges(queryWhere(2, "_from=2 and weight in (10, 20)")) - (result \ "results").as[List[JsValue]].size must equalTo(2) - } - } - - "get edge exclude" in { - running(FakeApplication()) { - val result = getEdges(queryExclude(0)) - (result \ "results").as[List[JsValue]].size must equalTo(1) - } - } - - "get edge groupBy property" in { - running(FakeApplication()) { - val result = getEdges(queryGroupBy(0, Seq("weight"))) - (result \ "size").as[Int] must_== 2 - val weights = (result \\ "groupBy").map { js => - (js \ "weight").as[Int] - } - weights must contain(exactly(30, 40)) - weights must not contain (10) - } - } - - "edge transform " in { - running(FakeApplication()) { - var result = getEdges(queryTransform(0, "[[\"_to\"]]")) - (result \ "results").as[List[JsValue]].size must equalTo(2) - - result = getEdges(queryTransform(0, "[[\"weight\"]]")) - (result \\ "to").map(_.toString).sorted must equalTo((result \\ "weight").map(_.toString).sorted) - - result = getEdges(queryTransform(0, "[[\"_from\"]]")) - val results = (result \ "results").as[JsValue] - (result \\ "to").map(_.toString).sorted must equalTo((results \\ "from").map(_.toString).sorted) - } - } - - "index" in { - running(FakeApplication()) { - // weight order - var result = getEdges(queryIndex(Seq(0), "idx_1")) - ((result \ "results").as[List[JsValue]].head \\ "weight").head must equalTo(JsNumber(40)) - - // timestamp order - result = getEdges(queryIndex(Seq(0), "idx_2")) - ((result \ "results").as[List[JsValue]].head \\ "weight").head must equalTo(JsNumber(30)) - } - } - - "checkEdges" in { - running(FakeApplication()) { - val json = Json.parse( s""" - [{"from": 0, "to": 1, "label": "$testLabelName"}, - {"from": 0, "to": 2, "label": "$testLabelName"}] - """) - - def checkEdges(queryJson: JsValue): JsValue = { - val ret = route(FakeRequest(POST, "/graphs/checkEdges").withJsonBody(queryJson)).get - contentAsJson(ret) - } - - val res = checkEdges(json) - val typeRes = res.isInstanceOf[JsArray] - typeRes must equalTo(true) - - val fst = res.as[Seq[JsValue]].head \ "to" - fst.as[Int] must equalTo(1) - - val snd = res.as[Seq[JsValue]].last \ "to" - snd.as[Int] must equalTo(2) - } - } - - "duration" in { - running(FakeApplication()) { - // get all - var result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000)) - (result \ "results").as[List[JsValue]].size must equalTo(4) - - // inclusive, exclusive - result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000)) - (result \ "results").as[List[JsValue]].size must equalTo(3) - - result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000)) - (result \ "results").as[List[JsValue]].size must equalTo(1) - - val bulkEdges: String = Seq( - edge"1001 insert e 0 1 $testLabelName"($(weight = 10, is_hidden = true)), - edge"2002 insert e 0 2 $testLabelName"($(weight = 20, is_hidden = false)), - edge"3003 insert e 2 0 $testLabelName"($(weight = 30)), - edge"4004 insert e 2 1 $testLabelName"($(weight = 40)) - ).mkString("\n") - - val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) - // duration test after udpate - // get all - result = getEdges(queryDuration(Seq(0, 2), from = 0, to = 5000)) - (result \ "results").as[List[JsValue]].size must equalTo(4) - - // inclusive, exclusive - result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 4000)) - (result \ "results").as[List[JsValue]].size must equalTo(3) - - result = getEdges(queryDuration(Seq(0, 2), from = 1000, to = 2000)) - (result \ "results").as[List[JsValue]].size must equalTo(1) - true - } - } - - "returnTree" in { - running(FakeApplication()) { - val src = 100 - val tgt = 200 - val labelName = testLabelName - - val bulkEdges: String = Seq( - edge"1001 insert e $src $tgt $labelName" - ).mkString("\n") - - val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) - - val result = getEdges(queryParents(src)) - - val parents = (result \ "results").as[Seq[JsValue]] - val ret = parents.forall { edge => (edge \ "parents").as[Seq[JsValue]].size == 1 } - ret must equalTo(true) - } - } - - "pagination and _to" in { - running(FakeApplication()) { - val src = System.currentTimeMillis().toInt - val labelName = testLabelName - val bulkEdges: String = Seq( - edge"1001 insert e $src 1 $labelName"($(weight = 10, is_hidden = true)), - edge"2002 insert e $src 2 $labelName"($(weight = 20, is_hidden = false)), - edge"3003 insert e $src 3 $labelName"($(weight = 30)), - edge"4004 insert e $src 4 $labelName"($(weight = 40)) - ).mkString("\n") - - val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) - - var result = getEdges(querySingle(src, offset = 0, limit = 2)) - println(result) - var edges = (result \ "results").as[List[JsValue]] - edges.size must equalTo(2) - (edges(0) \ "to").as[Long] must beEqualTo(4) - (edges(1) \ "to").as[Long] must beEqualTo(3) - - result = getEdges(querySingle(src, offset = 1, limit = 2)) - println(result) - edges = (result \ "results").as[List[JsValue]] - edges.size must equalTo(2) - (edges(0) \ "to").as[Long] must beEqualTo(3) - (edges(1) \ "to").as[Long] must beEqualTo(2) - - result = getEdges(querySingleWithTo(src, offset = 0, limit = -1, to = 1)) - println(result) - edges = (result \ "results").as[List[JsValue]] - edges.size must equalTo(1) - } - } - - "orderBy" >> { - running(FakeApplication()) { - // insert test set - val bulkEdges: String = Seq( - edge"1001 insert e 0 1 $testLabelName"($(weight = 10, is_hidden = true)), - edge"2002 insert e 0 2 $testLabelName"($(weight = 20, is_hidden = false)), - edge"3003 insert e 2 0 $testLabelName"($(weight = 30)), - edge"4004 insert e 2 1 $testLabelName"($(weight = 40)) - ).mkString("\n") - contentAsJson(EdgeController.mutateAndPublish(bulkEdges, withWait = true)) - - // get edges - val edges = getEdges(queryScore(0, Map("weight" -> 1))) - val orderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "DESC", "timestamp" -> "DESC")))) - val ascOrderByScore = getEdges(queryOrderBy(0, Map("weight" -> 1), Seq(Map("score" -> "ASC", "timestamp" -> "DESC")))) - - println(edges) - println(orderByScore) - println(ascOrderByScore) - - val edgesTo = edges \ "results" \\ "to" - val orderByTo = orderByScore \ "results" \\ "to" - val ascOrderByTo = ascOrderByScore \ "results" \\ "to" - - edgesTo must_== Seq(JsNumber(2), JsNumber(1)) - edgesTo must_== orderByTo - ascOrderByTo must_== Seq(JsNumber(1), JsNumber(2)) - edgesTo.reverse must_== ascOrderByTo - } - } - - "query with sampling" in { - running(FakeApplication()) { - val sampleSize = 2 - val testId = 22 - val bulkEdges = Seq( - edge"1442985659166 insert e $testId 122 $testLabelName", - edge"1442985659166 insert e $testId 222 $testLabelName", - edge"1442985659166 insert e $testId 322 $testLabelName", - - edge"1442985659166 insert e $testId 922 $testLabelName2", - edge"1442985659166 insert e $testId 222 $testLabelName2", - edge"1442985659166 insert e $testId 322 $testLabelName2", - - edge"1442985659166 insert e 122 1122 $testLabelName", - edge"1442985659166 insert e 122 1222 $testLabelName", - edge"1442985659166 insert e 122 1322 $testLabelName", - edge"1442985659166 insert e 222 2122 $testLabelName", - edge"1442985659166 insert e 222 2222 $testLabelName", - edge"1442985659166 insert e 222 2322 $testLabelName", - edge"1442985659166 insert e 322 3122 $testLabelName", - edge"1442985659166 insert e 322 3222 $testLabelName", - edge"1442985659166 insert e 322 3322 $testLabelName" - ) - - val req = FakeRequest(POST, "/graphs/edges/bulk").withBody(bulkEdges.mkString("\n")) - Await.result(route(req).get, HTTP_REQ_WAITING_TIME) - - Thread.sleep(asyncFlushInterval) - - - val result1 = getEdges(queryWithSampling(testId, sampleSize)) - println(Json.toJson(result1)) - (result1 \ "results").as[List[JsValue]].size must equalTo(scala.math.min(sampleSize, bulkEdges.size)) - - val result2 = getEdges(twoStepQueryWithSampling(testId, sampleSize)) - println(Json.toJson(result2)) - (result2 \ "results").as[List[JsValue]].size must equalTo(scala.math.min(sampleSize * sampleSize, bulkEdges.size * bulkEdges.size)) - - val result3 = getEdges(twoQueryWithSampling(testId, sampleSize)) - println(Json.toJson(result3)) - (result3 \ "results").as[List[JsValue]].size must equalTo(sampleSize + 3) // edges in testLabelName2 = 3 - } - } - } -} diff --git a/s2rest_play/test/controllers/SpecCommon.scala b/s2rest_play/test/controllers/SpecCommon.scala deleted file mode 100644 index 93648ce9..00000000 --- a/s2rest_play/test/controllers/SpecCommon.scala +++ /dev/null @@ -1,365 +0,0 @@ -package controllers - -import com.kakao.s2graph.core._ -import com.kakao.s2graph.core.mysqls._ -import org.specs2.mutable.Specification -import play.api.libs.json._ -import play.api.test.FakeApplication -import play.api.test.Helpers._ - -import scala.concurrent.Future -import scala.concurrent.duration._ -import scala.util.Random - -trait SpecCommon extends Specification { - sequential - object Helper { - - import org.json4s.native.Serialization - - type KV = Map[String, Any] - - import scala.language.dynamics - - def $aa[T](args: T*) = List($a(args: _ *)) - - def $a[T](args: T*) = args.toList - - object $ extends Dynamic { - def applyDynamicNamed(name: String)(args: (String, Any)*): Map[String, Any] = args.toMap - } - - implicit class anyMapOps(map: Map[String, Any]) { - def toJson: JsValue = { - val js = Serialization.write(map)(org.json4s.DefaultFormats) - Json.parse(js) - } - } - - implicit class S2Context(val sc: StringContext) { - def edge(args: Any*)(implicit map: Map[String, Any] = Map.empty): String = { - val parts = sc.s(args: _*).split("\\s") - assert(parts.length == 6) - (parts.toList :+ map.toJson.toString).mkString("\t") - } - } - - } - - val curTime = System.currentTimeMillis - val t1 = curTime + 0 - val t2 = curTime + 1 - val t3 = curTime + 2 - val t4 = curTime + 3 - val t5 = curTime + 4 - - protected val testServiceName = "s2graph" - protected val testLabelName = "s2graph_label_test" - protected val testLabelName2 = "s2graph_label_test_2" - protected val testLabelNameV1 = "s2graph_label_test_v1" - protected val testLabelNameWeak = "s2graph_label_test_weak" - protected val testColumnName = "user_id_test" - protected val testColumnType = "long" - protected val testTgtColumnName = "item_id_test" - protected val testHTableName = "test-htable" - protected val newHTableName = "new-htable" - - val NUM_OF_EACH_TEST = 100 - val HTTP_REQ_WAITING_TIME = Duration(300, SECONDS) - val asyncFlushInterval = 100 - - val createService = s"""{"serviceName" : "$testServiceName"}""" - val testLabelNameCreate = - s""" - { - "label": "$testLabelName", - "srcServiceName": "$testServiceName", - "srcColumnName": "$testColumnName", - "srcColumnType": "long", - "tgtServiceName": "$testServiceName", - "tgtColumnName": "$testColumnName", - "tgtColumnType": "long", - "indices": [ - {"name": "idx_1", "propNames": ["weight", "time", "is_hidden", "is_blocked"]}, - {"name": "idx_2", "propNames": ["_timestamp"]} - ], - "props": [ - { - "name": "time", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "weight", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "is_hidden", - "dataType": "boolean", - "defaultValue": false - }, - { - "name": "is_blocked", - "dataType": "boolean", - "defaultValue": false - } - ], - "consistencyLevel": "strong", - "schemaVersion": "v2", - "compressionAlgorithm": "gz", - "hTableName": "$testHTableName" - }""" - - val testLabelName2Create = - s""" - { - "label": "$testLabelName2", - "srcServiceName": "$testServiceName", - "srcColumnName": "$testColumnName", - "srcColumnType": "long", - "tgtServiceName": "$testServiceName", - "tgtColumnName": "$testTgtColumnName", - "tgtColumnType": "string", - "indices": [{"name": "idx_1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}], - "props": [ - { - "name": "time", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "weight", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "is_hidden", - "dataType": "boolean", - "defaultValue": false - }, - { - "name": "is_blocked", - "dataType": "boolean", - "defaultValue": false - } - ], - "consistencyLevel": "strong", - "isDirected": false, - "schemaVersion": "v3", - "compressionAlgorithm": "gz" - }""" - - val testLabelNameV1Create = - s""" - { - "label": "$testLabelNameV1", - "srcServiceName": "$testServiceName", - "srcColumnName": "$testColumnName", - "srcColumnType": "long", - "tgtServiceName": "$testServiceName", - "tgtColumnName": "${testTgtColumnName}_v1", - "tgtColumnType": "string", - "indices": [{"name": "idx_1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}], - "props": [ - { - "name": "time", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "weight", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "is_hidden", - "dataType": "boolean", - "defaultValue": false - }, - { - "name": "is_blocked", - "dataType": "boolean", - "defaultValue": false - } - ], - "consistencyLevel": "strong", - "isDirected": true, - "schemaVersion": "v1", - "compressionAlgorithm": "gz" - }""" - val testLabelNameWeakCreate = - s""" - { - "label": "$testLabelNameWeak", - "srcServiceName": "$testServiceName", - "srcColumnName": "$testColumnName", - "srcColumnType": "long", - "tgtServiceName": "$testServiceName", - "tgtColumnName": "$testTgtColumnName", - "tgtColumnType": "string", - "indices": [{"name": "idx_1", "propNames": ["time", "weight", "is_hidden", "is_blocked"]}], - "props": [ - { - "name": "time", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "weight", - "dataType": "long", - "defaultValue": 0 - }, - { - "name": "is_hidden", - "dataType": "boolean", - "defaultValue": false - }, - { - "name": "is_blocked", - "dataType": "boolean", - "defaultValue": false - } - ], - "consistencyLevel": "weak", - "isDirected": true, - "compressionAlgorithm": "gz" - }""" - - val vertexPropsKeys = List( - ("age", "int") - ) - - val createVertex = - s"""{ - "serviceName": "$testServiceName", - "columnName": "$testColumnName", - "columnType": "$testColumnType", - "props": [ - {"name": "is_active", "dataType": "boolean", "defaultValue": true}, - {"name": "phone_number", "dataType": "string", "defaultValue": "-"}, - {"name": "nickname", "dataType": "string", "defaultValue": ".."}, - {"name": "activity_score", "dataType": "float", "defaultValue": 0.0}, - {"name": "age", "dataType": "integer", "defaultValue": 0} - ] - }""" - - - val TS = System.currentTimeMillis() - - def queryJson(serviceName: String, columnName: String, labelName: String, id: String, dir: String, cacheTTL: Long = -1L) = { - val s = - s"""{ - "srcVertices": [ - { - "serviceName": "$serviceName", - "columnName": "$columnName", - "id": $id - } - ], - "steps": [ - [ - { - "label": "$labelName", - "direction": "$dir", - "offset": 0, - "limit": 10, - "cacheTTL": $cacheTTL - } - ] - ] - }""" - println(s) - Json.parse(s) - } - - def checkEdgeQueryJson(params: Seq[(String, String, String, String)]) = { - val arr = for { - (label, dir, from, to) <- params - } yield { - Json.obj("label" -> label, "direction" -> dir, "from" -> from, "to" -> to) - } - - val s = Json.toJson(arr) - println(s) - s - } - - def vertexQueryJson(serviceName: String, columnName: String, ids: Seq[Int]) = { - Json.parse( - s""" - |[ - |{"serviceName": "$serviceName", "columnName": "$columnName", "ids": [${ids.mkString(",")} - ]} - |] - """.stripMargin) - } - - def randomProps() = { - (for { - (propKey, propType) <- vertexPropsKeys - } yield { - propKey -> Random.nextInt(100) - }).toMap - } - - def vertexInsertsPayload(serviceName: String, columnName: String, ids: Seq[Int]): Seq[JsValue] = { - ids.map { id => - Json.obj("id" -> id, "props" -> randomProps, "timestamp" -> System.currentTimeMillis()) - } - } - - def commonCheck(rslt: Future[play.api.mvc.Result]): JsValue = { - status(rslt) must equalTo(OK) - contentType(rslt) must beSome.which(_ == "application/json") - val jsRslt = contentAsJson(rslt) - println("======") - println(jsRslt) - println("======") - jsRslt.as[JsObject].keys.contains("size") must equalTo(true) - (jsRslt \ "size").as[Int] must greaterThan(0) - jsRslt.as[JsObject].keys.contains("results") must equalTo(true) - val jsRsltsObj = jsRslt \ "results" - jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("from") must equalTo(true) - jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("to") must equalTo(true) - jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("_timestamp") must equalTo(true) - jsRsltsObj.as[JsArray].value(0).as[JsObject].keys.contains("props") must equalTo(true) - jsRslt - } - - def init() = { - running(FakeApplication()) { - println("[init start]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") - Management.deleteService(testServiceName) - - // 1. createService - val result = AdminController.createServiceInner(Json.parse(createService)) - println(s">> Service created : $createService, $result") - - val labelNames = Map(testLabelName -> testLabelNameCreate, - testLabelName2 -> testLabelName2Create, - testLabelNameV1 -> testLabelNameV1Create, - testLabelNameWeak -> testLabelNameWeakCreate) - - for { - (labelName, create) <- labelNames - } { - Management.deleteLabel(labelName) - Label.findByName(labelName, useCache = false) match { - case None => - AdminController.createLabelInner(Json.parse(create)) - case Some(label) => - println(s">> Label already exist: $create, $label") - } - } - - // 5. create vertex - vertexPropsKeys.map { case (key, keyType) => - Management.addVertexProp(testServiceName, testColumnName, key, keyType) - } - - println("[init end]: >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") - } - } -} - diff --git a/s2rest_play/test/controllers/StrongLabelDeleteSpec.scala b/s2rest_play/test/controllers/StrongLabelDeleteSpec.scala deleted file mode 100644 index 04ccc79e..00000000 --- a/s2rest_play/test/controllers/StrongLabelDeleteSpec.scala +++ /dev/null @@ -1,345 +0,0 @@ -package controllers - -import java.util.concurrent.TimeUnit - - -import com.kakao.s2graph.core.utils.logger -import play.api.libs.json._ -import play.api.test.Helpers._ -import play.api.test.{FakeApplication, FakeRequest} - -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, Future} -import scala.util.Random - -class StrongLabelDeleteSpec extends SpecCommon { - init() -// implicit val timeout = Timeout(Duration(20, TimeUnit.MINUTES)) - - def bulkEdges(startTs: Int = 0) = Seq( - Seq(startTs + 1, "insert", "e", "0", "1", testLabelName2, s"""{"time": 10}""").mkString("\t"), - Seq(startTs + 2, "insert", "e", "0", "1", testLabelName2, s"""{"time": 11}""").mkString("\t"), - Seq(startTs + 3, "insert", "e", "0", "1", testLabelName2, s"""{"time": 12}""").mkString("\t"), - Seq(startTs + 4, "insert", "e", "0", "2", testLabelName2, s"""{"time": 10}""").mkString("\t"), - Seq(startTs + 5, "insert", "e", "10", "20", testLabelName2, s"""{"time": 10}""").mkString("\t"), - Seq(startTs + 6, "insert", "e", "10", "21", testLabelName2, s"""{"time": 11}""").mkString("\t"), - Seq(startTs + 7, "insert", "e", "11", "20", testLabelName2, s"""{"time": 12}""").mkString("\t"), - Seq(startTs + 8, "insert", "e", "12", "20", testLabelName2, s"""{"time": 13}""").mkString("\t") - ).mkString("\n") - - def query(id: Long, serviceName: String = testServiceName, columnName: String = testColumnName, - labelName: String = testLabelName2, direction: String = "out") = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "$serviceName", - "columnName": "$columnName", - "id": $id - }], - "steps": [ - [ { - "label": "$labelName", - "direction": "${direction}", - "offset": 0, - "limit": -1, - "duplicate": "raw" - } - ]] - }""") - - def getEdges(queryJson: JsValue): JsValue = { -// implicit val timeout = Timeout(Duration(20, TimeUnit.MINUTES)) - - val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get - contentAsJson(ret) - } - - def getDegree(jsValue: JsValue): Long = { - ((jsValue \ "degrees") \\ "_degree").headOption.map(_.as[Long]).getOrElse(0L) - } - - - "strong label delete test" should { - running(FakeApplication()) { - // insert bulk and wait .. - val edges = bulkEdges() - println(edges) - val jsResult = contentAsJson(EdgeController.mutateAndPublish(edges, withWait = true)) - } - - "test strong consistency select" in { - running(FakeApplication()) { - var result = getEdges(query(0)) - println(result) - (result \ "results").as[List[JsValue]].size must equalTo(2) - result = getEdges(query(10)) - println(result) - (result \ "results").as[List[JsValue]].size must equalTo(2) - true - } - } - - "test strong consistency duration. insert -> delete -> insert" in { - running(FakeApplication()) { - val ts0 = 1 - val ts1 = 2 - val ts2 = 3 - - val edges = Seq( - Seq(5, "insert", "edge", "-10", "-20", testLabelName2).mkString("\t"), - Seq(10, "delete", "edge", "-10", "-20", testLabelName2).mkString("\t"), - Seq(20, "insert", "edge", "-10", "-20", testLabelName2).mkString("\t") - ).mkString("\n") - - val jsResult = contentAsJson(EdgeController.mutateAndPublish(edges, withWait = true)) - - val result = getEdges(query(-10)) - - println(result) - - true - } - } - - "test strong consistency deleteAll" in { - running(FakeApplication()) { - - val deletedAt = 100 - var result = getEdges(query(20, direction = "in", columnName = testTgtColumnName)) - println(result) - (result \ "results").as[List[JsValue]].size must equalTo(3) - - val json = Json.arr(Json.obj("label" -> testLabelName2, - "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt)) - println(json) - contentAsString(EdgeController.deleteAllInner(json, withWait = true)) - - result = getEdges(query(11, direction = "out")) - println(result) - (result \ "results").as[List[JsValue]].size must equalTo(0) - - result = getEdges(query(12, direction = "out")) - println(result) - (result \ "results").as[List[JsValue]].size must equalTo(0) - - result = getEdges(query(10, direction = "out")) - println(result) - // 10 -> out -> 20 should not be in result. - (result \ "results").as[List[JsValue]].size must equalTo(1) - (result \\ "to").size must equalTo(1) - (result \\ "to").head.as[String] must equalTo("21") - - result = getEdges(query(20, direction = "in", columnName = testTgtColumnName)) - println(result) - (result \ "results").as[List[JsValue]].size must equalTo(0) - - val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(startTs = deletedAt + 1), withWait = true)) - - result = getEdges(query(20, direction = "in", columnName = testTgtColumnName)) - println(result) - (result \ "results").as[List[JsValue]].size must equalTo(3) - - true - - } - } - } - - - "labelargeSet of contention" should { - val labelName = testLabelName2 - val maxTgtId = 10 - val batchSize = 10 - val testNum = 3 - val numOfBatch = 10 - - def testInner(startTs: Long, src: Long) = { - val labelName = testLabelName2 - val lastOps = Array.fill(maxTgtId)("none") - var currentTs = startTs - - val allRequests = for { - ith <- (0 until numOfBatch) - jth <- (0 until batchSize) - } yield { - currentTs += 1 - - val tgt = Random.nextInt(maxTgtId) - val op = if (Random.nextDouble() < 0.5) "delete" else "update" - - lastOps(tgt) = op - Seq(currentTs, op, "e", src, src + tgt, labelName, "{}").mkString("\t") - } - - - allRequests.foreach(println(_)) -// println(lastOps.count(op => op != "delete" && op != "none")) -// println(lastOps) -// -// Thread.sleep(1000) - - val futures = Random.shuffle(allRequests).grouped(batchSize).map { bulkRequest => - val bulkEdge = bulkRequest.mkString("\n") - EdgeController.mutateAndPublish(bulkEdge, withWait = true) - } - - Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) - - val expectedDegree = lastOps.count(op => op != "delete" && op != "none") - val queryJson = query(id = src) - val result = getEdges(queryJson) - val resultSize = (result \ "size").as[Long] - val resultDegree = getDegree(result) - - println(lastOps.toList) - println(result) - - val ret = resultDegree == expectedDegree && resultSize == resultDegree - if (!ret) System.err.println(s"[Contention Failed]: $resultDegree, $expectedDegree") - - (ret, currentTs) - } - - "update delete" in { - running(FakeApplication()) { - val ret = for { - i <- (0 until testNum) - } yield { - val src = System.currentTimeMillis() - - val (ret, last) = testInner(i, src) - ret must beEqualTo(true) - ret - } - - ret.forall(identity) - } - } - - "update delete 2" in { - running(FakeApplication()) { - - val src = System.currentTimeMillis() - var ts = 0L - - val ret = for { - i <- (0 until testNum) - } yield { - val (ret, lastTs) = testInner(ts, src) - val deletedAt = lastTs + 1 - val deletedAt2 = lastTs + 2 - ts = deletedAt2 + 1 // nex start ts - - ret must beEqualTo(true) - - - val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt)) - val deleteAllRequest2 = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt2)) - - val deleteRet = EdgeController.deleteAllInner(deleteAllRequest, withWait = true) - val deleteRet2 = EdgeController.deleteAllInner(deleteAllRequest2, withWait = true) - - - Await.result(deleteRet, Duration(20, TimeUnit.MINUTES)) - Await.result(deleteRet2, Duration(20, TimeUnit.MINUTES)) - - val result = getEdges(query(id = src)) - println(result) - - val resultEdges = (result \ "results").as[Seq[JsValue]] - resultEdges.isEmpty must beEqualTo(true) - - val degreeAfterDeleteAll = getDegree(result) - degreeAfterDeleteAll must beEqualTo(0) - true - } - - ret.forall(identity) - } - } - - /** this test stress out test on degree - * when contention is low but number of adjacent edges are large */ - "large degrees" in { - running(FakeApplication()) { - - - val labelName = testLabelName2 - val dir = "out" - val maxSize = 100 - val deleteSize = 10 - val numOfConcurrentBatch = 100 - val src = System.currentTimeMillis() - val tgts = (0 until maxSize).map { ith => src + ith } - val deleteTgts = Random.shuffle(tgts).take(deleteSize) - val insertRequests = tgts.map { tgt => - Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t") - } - val deleteRequests = deleteTgts.take(deleteSize).map { tgt => - Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t") - } - val allRequests = Random.shuffle(insertRequests ++ deleteRequests) - // val allRequests = insertRequests ++ deleteRequests - val futures = allRequests.grouped(numOfConcurrentBatch).map { requests => - EdgeController.mutateAndPublish(requests.mkString("\n"), withWait = true) - } - Await.result(Future.sequence(futures), Duration(20, TimeUnit.MINUTES)) - - val expectedDegree = insertRequests.size - deleteRequests.size - val queryJson = query(id = src) - val result = getEdges(queryJson) - val resultSize = (result \ "size").as[Long] - val resultDegree = getDegree(result) - - // println(result) - - val ret = resultSize == expectedDegree && resultDegree == resultSize - println(s"[MaxSize]: $maxSize") - println(s"[DeleteSize]: $deleteSize") - println(s"[ResultDegree]: $resultDegree") - println(s"[ExpectedDegree]: $expectedDegree") - println(s"[ResultSize]: $resultSize") - ret must beEqualTo(true) - } - } - - "deleteAll" in { - running(FakeApplication()) { - - val labelName = testLabelName2 - val dir = "out" - val maxSize = 100 - val deleteSize = 10 - val numOfConcurrentBatch = 100 - val src = System.currentTimeMillis() - val tgts = (0 until maxSize).map { ith => src + ith } - val deleteTgts = Random.shuffle(tgts).take(deleteSize) - val insertRequests = tgts.map { tgt => - Seq(tgt, "insert", "e", src, tgt, labelName, "{}", dir).mkString("\t") - } - val deleteRequests = deleteTgts.take(deleteSize).map { tgt => - Seq(tgt + 1000, "delete", "e", src, tgt, labelName, "{}", dir).mkString("\t") - } - val allRequests = Random.shuffle(insertRequests ++ deleteRequests) - // val allRequests = insertRequests ++ deleteRequests - val futures = allRequests.grouped(numOfConcurrentBatch).map { requests => - EdgeController.mutateAndPublish(requests.mkString("\n"), withWait = true) - } - Await.result(Future.sequence(futures), Duration(10, TimeUnit.MINUTES)) - - val deletedAt = System.currentTimeMillis() - val deleteAllRequest = Json.arr(Json.obj("label" -> labelName, "ids" -> Json.arr(src), "timestamp" -> deletedAt)) - - Await.result(EdgeController.deleteAllInner(deleteAllRequest, withWait = true), Duration(10, TimeUnit.MINUTES)) - - val result = getEdges(query(id = src)) - println(result) - val resultEdges = (result \ "results").as[Seq[JsValue]] - resultEdges.isEmpty must beEqualTo(true) - - val degreeAfterDeleteAll = getDegree(result) - degreeAfterDeleteAll must beEqualTo(0) - } - } - } -} - diff --git a/s2rest_play/test/controllers/VertexSpec.scala b/s2rest_play/test/controllers/VertexSpec.scala deleted file mode 100644 index f427b578..00000000 --- a/s2rest_play/test/controllers/VertexSpec.scala +++ /dev/null @@ -1,39 +0,0 @@ -package controllers - -import play.api.libs.json._ -import play.api.test.FakeApplication -import play.api.test.Helpers._ - -class VertexSpec extends SpecCommon { - // init() - - "vetex tc" should { - "tc1" in { - - running(FakeApplication()) { - val ids = (7 until 20).map(tcNum => tcNum * 1000 + 0) - - val (serviceName, columnName) = (testServiceName, testColumnName) - - val data = vertexInsertsPayload(serviceName, columnName, ids) - val payload = Json.parse(Json.toJson(data).toString) - println(payload) - - val jsResult = contentAsString(VertexController.tryMutates(payload, "insert", - Option(serviceName), Option(columnName), withWait = true)) - - val query = vertexQueryJson(serviceName, columnName, ids) - val ret = contentAsJson(QueryController.getVerticesInner(query)) - println(">>>", ret) - val fetched = ret.as[Seq[JsValue]] - for { - (d, f) <- data.zip(fetched) - } yield { - (d \ "id") must beEqualTo((f \ "id")) - ((d \ "props") \ "age") must beEqualTo(((f \ "props") \ "age")) - } - } - true - } - } -} diff --git a/s2rest_play/test/controllers/WeakLabelDeleteSpec.scala b/s2rest_play/test/controllers/WeakLabelDeleteSpec.scala deleted file mode 100644 index 164c52bc..00000000 --- a/s2rest_play/test/controllers/WeakLabelDeleteSpec.scala +++ /dev/null @@ -1,126 +0,0 @@ -package controllers - -import play.api.libs.json._ -import play.api.test.Helpers._ -import play.api.test.{FakeApplication, FakeRequest} - -class WeakLabelDeleteSpec extends SpecCommon { - init() - - def bulkEdges(startTs: Int = 0) = Seq( - Seq(startTs + 1, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"), - Seq(startTs + 2, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 11}""").mkString("\t"), - Seq(startTs + 3, "insert", "e", "0", "1", testLabelNameWeak, s"""{"time": 12}""").mkString("\t"), - Seq(startTs + 4, "insert", "e", "0", "2", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"), - Seq(startTs + 5, "insert", "e", "10", "20", testLabelNameWeak, s"""{"time": 10}""").mkString("\t"), - Seq(startTs + 6, "insert", "e", "10", "21", testLabelNameWeak, s"""{"time": 11}""").mkString("\t"), - Seq(startTs + 7, "insert", "e", "11", "20", testLabelNameWeak, s"""{"time": 12}""").mkString("\t"), - Seq(startTs + 8, "insert", "e", "12", "20", testLabelNameWeak, s"""{"time": 13}""").mkString("\t") - ).mkString("\n") - - "weak label delete test" should { - running(FakeApplication()) { - // insert bulk and wait .. - val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(), withWait = true)) - } - - - def query(id: Int, direction: String = "out", columnName: String = testColumnName) = Json.parse( - s""" - { "srcVertices": [ - { "serviceName": "$testServiceName", - "columnName": "$columnName", - "id": ${id} - }], - "steps": [ - [ { - "label": "${testLabelNameWeak}", - "direction": "${direction}", - "offset": 0, - "limit": 10, - "duplicate": "raw" - } - ]] - }""") - - def getEdges(queryJson: JsValue): JsValue = { - val ret = route(FakeRequest(POST, "/graphs/getEdges").withJsonBody(queryJson)).get - contentAsJson(ret) - } - - "test weak consistency select" in { - running(FakeApplication()) { - var result = getEdges(query(0)) - println(result) - (result \ "results").as[List[JsValue]].size must equalTo(4) - result = getEdges(query(10)) - println(result) - (result \ "results").as[List[JsValue]].size must equalTo(2) - true - } - } - - "test weak consistency delete" in { - running(FakeApplication()) { - var result = getEdges(query(0)) - println(result) - - /** expect 4 edges */ - (result \ "results").as[List[JsValue]].size must equalTo(4) - val edges = (result \ "results").as[List[JsObject]] - - contentAsJson(EdgeController.tryMutates(Json.toJson(edges), "delete", withWait = true)) - - /** expect noting */ - result = getEdges(query(0)) - println(result) - (result \ "results").as[List[JsValue]].size must equalTo(0) - - /** insert should be ignored */ - contentAsJson(EdgeController.tryMutates(Json.toJson(edges), "insert", withWait = true)) - - result = getEdges(query(0)) - (result \ "results").as[List[JsValue]].size must equalTo(0) - } - } - - "test weak consistency deleteAll" in { - running(FakeApplication()) { - val deletedAt = 100 - var result = getEdges(query(20, "in", testTgtColumnName)) - println(result) - (result \ "results").as[List[JsValue]].size must equalTo(3) - - val json = Json.arr(Json.obj("label" -> testLabelNameWeak, - "direction" -> "in", "ids" -> Json.arr("20"), "timestamp" -> deletedAt)) - println(json) - contentAsString(EdgeController.deleteAllInner(json, withWait = true)) - - - result = getEdges(query(11, "out")) - (result \ "results").as[List[JsValue]].size must equalTo(0) - - result = getEdges(query(12, "out")) - (result \ "results").as[List[JsValue]].size must equalTo(0) - - result = getEdges(query(10, "out")) - - // 10 -> out -> 20 should not be in result. - (result \ "results").as[List[JsValue]].size must equalTo(1) - (result \\ "to").size must equalTo(1) - (result \\ "to").head.as[String] must equalTo("21") - - result = getEdges(query(20, "in", testTgtColumnName)) - println(result) - (result \ "results").as[List[JsValue]].size must equalTo(0) - - val jsResult = contentAsJson(EdgeController.mutateAndPublish(bulkEdges(startTs = deletedAt + 1), withWait = true)) - - result = getEdges(query(20, "in", testTgtColumnName)) - (result \ "results").as[List[JsValue]].size must equalTo(3) - } - } - } - -} - From 8b3ddf64cc47e9270cfbe0d4295a5f590e05186c Mon Sep 17 00:00:00 2001 From: daewon Date: Thu, 14 Jan 2016 18:13:29 +0900 Subject: [PATCH 06/13] S2GRAPH-7 Apply review by SteamShon --- s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala | 2 +- .../test/scala/com/kakao/s2graph/core/types/InnerValTest.scala | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala b/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala index 5fc44ccd..111f3df1 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/Graph.scala @@ -312,7 +312,7 @@ object Graph { } get } -class Graph(_config: Config)(implicit val ec: ExecutionContext) { +class Graph(_config: Config)(implicit ec: ExecutionContext) { val config = _config.withFallback(Graph.DefaultConfig) val cacheSize = config.getInt("cache.max.size") // val cache = CacheBuilder.newBuilder().maximumSize(cacheSize).build[java.lang.Integer, Seq[QueryResult]]() diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala index 9e01da3f..69a299de 100644 --- a/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala +++ b/s2core/src/test/scala/com/kakao/s2graph/core/types/InnerValTest.scala @@ -8,6 +8,7 @@ import play.api.libs.json.Json class InnerValTest extends FunSuite with Matchers with TestCommonWithModels { initTests() + import HBaseType.{VERSION2, VERSION1} val decimals = List( BigDecimal(Long.MinValue), From 964bbc4988ab4e84b7efe6f6fdcc8be9a0e1ee7c Mon Sep 17 00:00:00 2001 From: daewon Date: Wed, 6 Jan 2016 18:19:54 +0900 Subject: [PATCH 07/13] S2GRAPH-9 Provide rest server using netty --- build.sbt | 4 + .../s2graph/core/mysqls/Experiment.scala | 5 +- .../kakao/s2graph/core/utils/Extentions.scala | 7 + .../src/main/resources/reference.conf | 68 ++++ s2rest_netty/src/main/scala/Server.scala | 213 ++++++++++++ s2rest_play/app/Bootstrap.scala | 4 +- .../controllers/ApplicationController.scala | 17 + .../controllers/ExperimentController.scala | 107 +----- .../app/controllers/QueryController.scala | 311 ++---------------- 9 files changed, 346 insertions(+), 390 deletions(-) create mode 100644 s2rest_netty/src/main/resources/reference.conf create mode 100644 s2rest_netty/src/main/scala/Server.scala diff --git a/build.sbt b/build.sbt index 30e5b644..5efa284c 100755 --- a/build.sbt +++ b/build.sbt @@ -26,6 +26,10 @@ lazy val s2rest_play = project.enablePlugins(PlayScala) .settings(commonSettings: _*) .settings(testOptions in Test += Tests.Argument("sequential")) +lazy val s2rest_netty = project + .dependsOn(s2core) + .settings(commonSettings: _*) + lazy val s2core = project.settings(commonSettings: _*) lazy val spark = project.settings(commonSettings: _*) diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Experiment.scala b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Experiment.scala index 93cc374f..d4849147 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Experiment.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/mysqls/Experiment.scala @@ -5,10 +5,9 @@ import scalikejdbc._ import scala.util.Random -/** - * Created by shon on 8/5/15. - */ object Experiment extends Model[Experiment] { + val impressionKey = "S2-Impression-Id" + def apply(rs: WrappedResultSet): Experiment = { Experiment(rs.intOpt("id"), rs.int("service_id"), diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/utils/Extentions.scala b/s2core/src/main/scala/com/kakao/s2graph/core/utils/Extentions.scala index 3812c2c0..4858f60e 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/utils/Extentions.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/utils/Extentions.scala @@ -1,6 +1,8 @@ package com.kakao.s2graph.core.utils import com.stumbleupon.async.{Callback, Deferred} +import com.typesafe.config.Config + import scala.concurrent.{ExecutionContext, Future, Promise} object Extensions { @@ -64,4 +66,9 @@ object Extensions { } + implicit class ConfigOps(config: Config) { + def getBooleanWithFallback(key: String, defaultValue: Boolean): Boolean = + if (config.hasPath(key)) config.getBoolean(key) else defaultValue + } + } diff --git a/s2rest_netty/src/main/resources/reference.conf b/s2rest_netty/src/main/resources/reference.conf new file mode 100644 index 00000000..7136b189 --- /dev/null +++ b/s2rest_netty/src/main/resources/reference.conf @@ -0,0 +1,68 @@ +# APP PHASE +phase=dev +host=localhost + +# DB +s2graph.models.table.name="models-dev" +hbase.zookeeper.quorum=${host} +db.default.url="jdbc:mysql://"${host}":3306/graph_dev" +# Query server +is.query.server=true +is.write.server=true +query.hard.limit=100000 + +# Local Cache +cache.ttl.seconds=60 +cache.max.size=100000 + +# HBASE +#hbase.client.operation.timeout=1000 +#async.hbase.client.flush.interval=100 +hbase.table.compression.algorithm="gz" + +# Asynchbase +hbase.client.retries.number=100 +hbase.rpcs.buffered_flush_interval=100 +hbase.rpc.timeout=0 +#hbase.nsre.high_watermark=1000000 +#hbase.timer.tick=5 +#hbase.timer.ticks_per_wheel=5 + +# Kafka +kafka.metadata.broker.list=${host} +kafka.producer.pool.size=0 + +# HTTP +parsers.text.maxLength=512K +parsers.json.maxLength=512K +trustxforwarded=false + +# Local Queue Actor +local.queue.actor.max.queue.size=100000 +local.queue.actor.rate.limit=1000000 + +# local retry number +max.retry.number=100 +max.back.off=50 +delete.all.fetch.size=10000 +hbase.fail.prob=-1.0 + +# max allowd edges for deleteAll is multiply of above two configuration. + +# set global obejct package, TODO: remove global +application.global=com.kakao.s2graph.rest.Global + +akka { + loggers = ["akka.event.slf4j.Slf4jLogger"] + loglevel = "DEBUG" +} + +# Future cache. +future.cache.max.size=100000 +future.cache.expire.after.write=10000 +future.cache.expire.after.access=5000 + + +# Counter +redis.instances = [${host}] + diff --git a/s2rest_netty/src/main/scala/Server.scala b/s2rest_netty/src/main/scala/Server.scala new file mode 100644 index 00000000..10d12751 --- /dev/null +++ b/s2rest_netty/src/main/scala/Server.scala @@ -0,0 +1,213 @@ +package com.kakao.s2graph.rest.netty + +import java.util.concurrent.Executors + +import com.kakao.s2graph.core.GraphExceptions.BadQueryException +import com.kakao.s2graph.core._ +import com.kakao.s2graph.core.mysqls.Experiment +import com.kakao.s2graph.core.rest.RestCaller +import com.kakao.s2graph.core.utils.Extensions._ +import com.kakao.s2graph.core.utils.logger +import com.typesafe.config.ConfigFactory +import io.netty.bootstrap.ServerBootstrap +import io.netty.buffer.{ByteBuf, Unpooled} +import io.netty.channel._ +import io.netty.channel.nio.NioEventLoopGroup +import io.netty.channel.socket.SocketChannel +import io.netty.channel.socket.nio.NioServerSocketChannel +import io.netty.handler.codec.http.HttpHeaders._ +import io.netty.handler.codec.http._ +import io.netty.handler.logging.{LogLevel, LoggingHandler} +import io.netty.util.CharsetUtil +import play.api.libs.json._ + +import scala.concurrent.{ExecutionContext, Future} +import scala.io.Source +import scala.util.{Failure, Success, Try} + +class S2RestHandler(s2rest: RestCaller)(implicit ec: ExecutionContext) extends SimpleChannelInboundHandler[FullHttpRequest] with JSONParser { + val ApplicationJson = "application/json" + + val Ok = HttpResponseStatus.OK + val Close = ChannelFutureListener.CLOSE + val BadRequest = HttpResponseStatus.BAD_REQUEST + val BadGateway = HttpResponseStatus.BAD_GATEWAY + val NotFound = HttpResponseStatus.NOT_FOUND + val InternalServerError = HttpResponseStatus.INTERNAL_SERVER_ERROR + + def badRoute(ctx: ChannelHandlerContext) = + simpleResponse(ctx, BadGateway, byteBufOpt = None, channelFutureListenerOpt = Option(Close)) + + def simpleResponse(ctx: ChannelHandlerContext, + httpResponseStatus: HttpResponseStatus, + byteBufOpt: Option[ByteBuf] = None, + headers: Seq[(String, String)] = Nil, + channelFutureListenerOpt: Option[ChannelFutureListener] = None): Unit = { + + val res: FullHttpResponse = byteBufOpt match { + case None => new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, httpResponseStatus) + case Some(byteBuf) => + new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, httpResponseStatus, byteBuf) + } + + headers.foreach { case (k, v) => res.headers().set(k, v) } + val channelFuture = ctx.writeAndFlush(res) + + channelFutureListenerOpt match { + case None => + case Some(listener) => channelFuture.addListener(listener) + } + } + + def toResponse(ctx: ChannelHandlerContext, req: FullHttpRequest, body: JsValue, future: Future[(JsValue, String)], startedAt: Long) = { + + val defaultHeaders = List(Names.CONTENT_TYPE -> ApplicationJson) + // NOTE: logging size of result should move to s2core. + // logger.info(resJson.size.toString) + future onComplete { + case Success(resJsonWithImpId) => + val (resJson, impId) = resJsonWithImpId + + val duration = System.currentTimeMillis() - startedAt + val isKeepAlive = HttpHeaders.isKeepAlive(req) + val buf: ByteBuf = Unpooled.copiedBuffer(resJson.toString, CharsetUtil.UTF_8) + + + val (headerWithKeepAlive, listenerOpt) = + if (isKeepAlive) ((Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE) :: defaultHeaders) -> None + else defaultHeaders -> Option(Close) + + val headerWithImpKey = + if (impId.isEmpty) headerWithKeepAlive + else (Experiment.impressionKey, impId) :: headerWithKeepAlive + + val log = s"${req.getMethod} ${req.getUri} took ${duration} ms 200 ${s2rest.calcSize(resJson)} ${body}" + logger.info(log) + + val responseHeaders = (Names.CONTENT_LENGTH -> buf.readableBytes().toString) :: headerWithImpKey + simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = listenerOpt, headers = responseHeaders) + + case Failure(ex) => ex match { + case e: BadQueryException => + logger.error(s"{$body}, ${e.getMessage}", e) + val buf: ByteBuf = Unpooled.copiedBuffer(PostProcess.badRequestResults(e).toString, CharsetUtil.UTF_8) + simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = Option(Close), headers = defaultHeaders) + case e: Exception => + logger.error(s"${body}, ${e.getMessage}", e) + val buf: ByteBuf = Unpooled.copiedBuffer(PostProcess.emptyResults.toString, CharsetUtil.UTF_8) + simpleResponse(ctx, InternalServerError, byteBufOpt = Option(buf), channelFutureListenerOpt = Option(Close), headers = defaultHeaders) + } + } + } + + override def channelRead0(ctx: ChannelHandlerContext, req: FullHttpRequest): Unit = { + val uri = req.getUri + + req.getMethod match { + case HttpMethod.GET => + uri match { + case "/health_check.html" => + if (NettyServer.isHealthy) { + val healthCheckMsg = Unpooled.copiedBuffer(NettyServer.deployInfo, CharsetUtil.UTF_8) + simpleResponse(ctx, Ok, byteBufOpt = Option(healthCheckMsg), channelFutureListenerOpt = Option(Close)) + } else { + simpleResponse(ctx, NotFound, channelFutureListenerOpt = Option(Close)) + } + + case s if s.startsWith("/graphs/getEdge/") => + // src, tgt, label, dir + val Array(srcId, tgtId, labelName, direction) = s.split("/").takeRight(4) + val params = Json.arr(Json.obj("label" -> labelName, "direction" -> direction, "from" -> srcId, "to" -> tgtId)) + val startedAt = System.currentTimeMillis() + val future = s2rest.checkEdges(params) + toResponse(ctx, req, params, future.map(_ -> ""), startedAt) + case _ => badRoute(ctx) + } + + case HttpMethod.PUT => + if (uri.startsWith("/health_check/")) { + val newHealthCheck = uri.split("/").last.toBoolean + NettyServer.isHealthy = newHealthCheck + val newHealthCheckMsg = Unpooled.copiedBuffer(NettyServer.isHealthy.toString, CharsetUtil.UTF_8) + simpleResponse(ctx, Ok, byteBufOpt = Option(newHealthCheckMsg), channelFutureListenerOpt = Option(Close)) + } else badRoute(ctx) + + case HttpMethod.POST => + val jsonString = req.content.toString(CharsetUtil.UTF_8) + val jsQuery = Json.parse(jsonString) + + //TODO: result_size + val startedAt = System.currentTimeMillis() + + val future = + if (uri.startsWith("/graphs/experiment")) { + val Array(accessToken, experimentName, uuid) = uri.split("/").takeRight(3) + s2rest.experiment(jsQuery, accessToken, experimentName, uuid) + } else { + s2rest.uriMatch(uri, jsQuery).map(_ -> "") + } + + toResponse(ctx, req, jsQuery, future, startedAt) + + case _ => + simpleResponse(ctx, BadRequest, byteBufOpt = None, channelFutureListenerOpt = Option(Close)) + } + } + + override def exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { + cause.printStackTrace() + logger.error(s"exception on query.", cause) + simpleResponse(ctx, BadRequest, byteBufOpt = None, channelFutureListenerOpt = Option(Close)) + } +} + +// Simple http server +object NettyServer extends App { + /** should be same with Boostrap.onStart on play */ + + val numOfThread = Runtime.getRuntime.availableProcessors() + val threadPool = Executors.newFixedThreadPool(numOfThread) + val ec = ExecutionContext.fromExecutor(threadPool) + + val config = ConfigFactory.load() + val port = Try(config.getInt("http.port")).recover { case _ => 9000 }.get + + // init s2graph with config + val s2graph = new Graph(config)(ec) + val rest = new RestCaller(s2graph)(ec) + + val deployInfo = Try(Source.fromFile("./release_info").mkString("")).recover { case _ => "release info not found\n" }.get + var isHealthy = config.getBooleanWithFallback("app.health.on", true) + + logger.info(s"starts with num of thread: $numOfThread, ${threadPool.getClass.getSimpleName}") + + // Configure the server. + val bossGroup: EventLoopGroup = new NioEventLoopGroup(1) + val workerGroup: EventLoopGroup = new NioEventLoopGroup() + + try { + val b: ServerBootstrap = new ServerBootstrap() + b.option(ChannelOption.SO_BACKLOG, Int.box(2048)) + + b.group(bossGroup, workerGroup).channel(classOf[NioServerSocketChannel]) + .handler(new LoggingHandler(LogLevel.INFO)) + .childHandler(new ChannelInitializer[SocketChannel] { + override def initChannel(ch: SocketChannel) { + val p = ch.pipeline() + p.addLast(new HttpServerCodec()) + p.addLast(new HttpObjectAggregator(65536)) + p.addLast(new S2RestHandler(rest)(ec)) + } + }) + + logger.info(s"Listening for HTTP on /0.0.0.0:$port") + val ch: Channel = b.bind(port).sync().channel() + ch.closeFuture().sync() + + } finally { + bossGroup.shutdownGracefully() + workerGroup.shutdownGracefully() + s2graph.shutdown() + } +} + diff --git a/s2rest_play/app/Bootstrap.scala b/s2rest_play/app/Bootstrap.scala index a661ac3e..bc883f83 100644 --- a/s2rest_play/app/Bootstrap.scala +++ b/s2rest_play/app/Bootstrap.scala @@ -3,7 +3,7 @@ package com.kakao.s2graph.rest import java.util.concurrent.Executors import actors.QueueActor -import com.kakao.s2graph.core.rest.RequestParser +import com.kakao.s2graph.core.rest.{RestCaller, RequestParser} import com.kakao.s2graph.core.utils.logger import com.kakao.s2graph.core.{ExceptionHandler, Graph} import config.Config @@ -19,6 +19,7 @@ import scala.util.Try object Global extends WithFilters(new GzipFilter()) { var s2graph: Graph = _ var s2parser: RequestParser = _ + var s2rest: RestCaller = _ // Application entry point override def onStart(app: Application) { @@ -33,6 +34,7 @@ object Global extends WithFilters(new GzipFilter()) { // init s2graph with config s2graph = new Graph(config)(ec) s2parser = new RequestParser(s2graph.config) // merged config + s2rest = new RestCaller(s2graph)(ec) QueueActor.init(s2graph) diff --git a/s2rest_play/app/controllers/ApplicationController.scala b/s2rest_play/app/controllers/ApplicationController.scala index fefe93e3..d9384d96 100644 --- a/s2rest_play/app/controllers/ApplicationController.scala +++ b/s2rest_play/app/controllers/ApplicationController.scala @@ -1,5 +1,7 @@ package controllers +import com.kakao.s2graph.core.GraphExceptions.BadQueryException +import com.kakao.s2graph.core.PostProcess import com.kakao.s2graph.core.utils.logger import play.api.libs.iteratee.Enumerator import play.api.libs.json.{JsString, JsValue} @@ -15,6 +17,21 @@ object ApplicationController extends Controller { val jsonParser: BodyParser[JsValue] = controllers.s2parse.json + private def badQueryExceptionResults(ex: Exception) = + Future.successful(BadRequest(PostProcess.badRequestResults(ex)).as(applicationJsonHeader)) + + private def errorResults = + Future.successful(Ok(PostProcess.emptyResults).as(applicationJsonHeader)) + + def requestFallback(body: JsValue): PartialFunction[Throwable, Future[Result]] = { + case e: BadQueryException => + logger.error(s"{$body}, ${e.getMessage}", e) + badQueryExceptionResults(e) + case e: Exception => + logger.error(s"${body}, ${e.getMessage}", e) + errorResults + } + def updateHealthCheck(isHealthy: Boolean) = Action { request => this.isHealthy = isHealthy Ok(this.isHealthy + "\n") diff --git a/s2rest_play/app/controllers/ExperimentController.scala b/s2rest_play/app/controllers/ExperimentController.scala index 5ae2379e..a4b5a17b 100644 --- a/s2rest_play/app/controllers/ExperimentController.scala +++ b/s2rest_play/app/controllers/ExperimentController.scala @@ -1,114 +1,23 @@ package controllers -import java.net.URL - -import com.kakao.s2graph.core.mysqls._ -import com.kakao.s2graph.core.rest.RequestParser -import com.kakao.s2graph.core.utils.logger -import play.api.Play.current -import play.api.libs.json.{JsObject, JsString, JsValue, Json} -import play.api.libs.ws.WS +import com.kakao.s2graph.core.mysqls.Experiment +import com.kakao.s2graph.core.rest.RestCaller import play.api.mvc._ import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future object ExperimentController extends Controller { - val impressionKey = "S2-Impression-Id" + private val rest: RestCaller = com.kakao.s2graph.rest.Global.s2rest import ApplicationController._ def experiment(accessToken: String, experimentName: String, uuid: String) = withHeaderAsync(parse.anyContent) { request => - val bucketOpt = for { - service <- Service.findByAccessToken(accessToken) - experiment <- Experiment.findBy(service.id.get, experimentName) - bucket <- experiment.findBucket(uuid) - } yield bucket - - bucketOpt match { - case None => Future.successful(NotFound("bucket is not found.")) - case Some(bucket) => - try { - if (bucket.isGraphQuery) buildRequestInner(request, bucket, uuid) - else buildRequest(request, bucket, uuid) - } catch { - case e: Exception => - logger.error(e.toString()) - Future.successful(BadRequest(s"wrong or missing template parameter: ${e.getMessage}")) - } - } - } - - def makeRequestJson(requestKeyJsonOpt: Option[JsValue], bucket: Bucket, uuid: String): JsValue = { - var body = bucket.requestBody.replace("#uuid", uuid) - - for { - requestKeyJson <- requestKeyJsonOpt - jsObj <- requestKeyJson.asOpt[JsObject] - (key, value) <- jsObj.fieldSet - } { - val replacement = value match { - case JsString(s) => s - case _ => value.toString - } - body = body.replace(key, replacement) - } - - Json.parse(body) - } - - private def buildRequestInner(request: Request[AnyContent], bucket: Bucket, uuid: String): Future[Result] = { - if (bucket.isEmpty) Future.successful(Ok(Json.obj("isEmpty" -> true)).withHeaders(impressionKey -> bucket.impressionId)) - else { - val jsonBody = makeRequestJson(request.body.asJson, bucket, uuid) - val url = new URL(bucket.apiPath) - val path = url.getPath() - - // dummy log for sampling - val experimentLog = s"POST $path took -1 ms 200 -1 $jsonBody" - logger.info(experimentLog) - - val response = path match { - case "/graphs/getEdges" => controllers.QueryController.getEdgesInner(jsonBody) - case "/graphs/getEdges/grouped" => controllers.QueryController.getEdgesWithGroupingInner(jsonBody) - case "/graphs/getEdgesExcluded" => controllers.QueryController.getEdgesExcludedInner(jsonBody) - case "/graphs/getEdgesExcluded/grouped" => controllers.QueryController.getEdgesExcludedWithGroupingInner(jsonBody) - case "/graphs/checkEdges" => controllers.QueryController.checkEdgesInner(jsonBody) - case "/graphs/getEdgesGrouped" => controllers.QueryController.getEdgesGroupedInner(jsonBody) - case "/graphs/getEdgesGroupedExcluded" => controllers.QueryController.getEdgesGroupedExcludedInner(jsonBody) - case "/graphs/getEdgesGroupedExcludedFormatted" => controllers.QueryController.getEdgesGroupedExcludedFormattedInner(jsonBody) - } - response.map { r => r.withHeaders(impressionKey -> bucket.impressionId) } - } - } - - private def toSimpleMap(map: Map[String, Seq[String]]): Map[String, String] = { - for { - (k, vs) <- map - headVal <- vs.headOption - } yield { - k -> headVal - } - } - - private def buildRequest(request: Request[AnyContent], bucket: Bucket, uuid: String): Future[Result] = { - val jsonBody = makeRequestJson(request.body.asJson, bucket, uuid) - - val url = bucket.apiPath - val headers = request.headers.toSimpleMap.toSeq - val verb = bucket.httpVerb.toUpperCase - val qs = toSimpleMap(request.queryString).toSeq - - val ws = WS.url(url) - .withMethod(verb) - .withBody(jsonBody) - .withHeaders(headers: _*) - .withQueryString(qs: _*) + val body = request.body.asJson.get - ws.stream().map { - case (proxyResponse, proxyBody) => - Result(ResponseHeader(proxyResponse.status, proxyResponse.headers.mapValues(_.toList.head)), proxyBody).withHeaders(impressionKey -> bucket.impressionId) - } + val res = rest.experiment(body, accessToken, experimentName, uuid) + res.map { case (js, impId) => + jsonResponse(js, Experiment.impressionKey -> impId, "result_size" -> rest.calcSize(js).toString) + } recoverWith ApplicationController.requestFallback(body) } } diff --git a/s2rest_play/app/controllers/QueryController.scala b/s2rest_play/app/controllers/QueryController.scala index 480c142c..218d76e9 100644 --- a/s2rest_play/app/controllers/QueryController.scala +++ b/s2rest_play/app/controllers/QueryController.scala @@ -1,311 +1,48 @@ package controllers -import com.kakao.s2graph.core.GraphExceptions.BadQueryException import com.kakao.s2graph.core._ -import com.kakao.s2graph.core.mysqls._ -import com.kakao.s2graph.core.types.{LabelWithDirection, VertexId} -import com.kakao.s2graph.core.utils.logger -import config.Config -import play.api.libs.json.{JsArray, JsObject, JsValue, Json} -import play.api.mvc.{Action, Controller, Result} +import com.kakao.s2graph.core.rest.RestCaller +import play.api.libs.json.{JsValue, Json} +import play.api.mvc.{Controller, Request} -import scala.concurrent._ import scala.language.postfixOps -import scala.util.Try -object QueryController extends Controller { +object QueryController extends Controller with JSONParser { import ApplicationController._ import play.api.libs.concurrent.Execution.Implicits.defaultContext - private val s2: Graph = com.kakao.s2graph.rest.Global.s2graph - private val requestParser = com.kakao.s2graph.rest.Global.s2parser + private val rest: RestCaller = com.kakao.s2graph.rest.Global.s2rest - private def badQueryExceptionResults(ex: Exception) = Future.successful(BadRequest(Json.obj("message" -> ex.getMessage)).as(applicationJsonHeader)) + def delegate(request: Request[JsValue]) = + rest.uriMatch(request.uri, request.body).map { js => + jsonResponse(js, "result_size" -> rest.calcSize(js).toString) + } recoverWith ApplicationController.requestFallback(request.body) - private def errorResults = Future.successful(Ok(PostProcess.timeoutResults).as(applicationJsonHeader)) + def getEdges() = withHeaderAsync(jsonParser)(delegate) - def getEdges() = withHeaderAsync(jsonParser) { request => - getEdgesInner(request.body) - } + def getEdgesWithGrouping() = withHeaderAsync(jsonParser)(delegate) - def getEdgesExcluded = withHeaderAsync(jsonParser) { request => - getEdgesExcludedInner(request.body) - } + def getEdgesExcluded() = withHeaderAsync(jsonParser)(delegate) - private def eachQuery(post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue)(q: Query): Future[JsValue] = { - val filterOutQueryResultsLs = q.filterOutQuery match { - case Some(filterOutQuery) => s2.getEdges(filterOutQuery) - case None => Future.successful(Seq.empty) - } - - for { - queryResultsLs <- s2.getEdges(q) - filterOutResultsLs <- filterOutQueryResultsLs - } yield { - val json = post(queryResultsLs, filterOutResultsLs) - json - } - } - - private def calcSize(js: JsValue): Int = js match { - case JsObject(obj) => (js \ "size").asOpt[Int].getOrElse(0) - case JsArray(seq) => seq.map(js => (js \ "size").asOpt[Int].getOrElse(0)).sum - case _ => 0 - } - - private def getEdgesAsync(jsonQuery: JsValue) - (post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue): Future[Result] = { - if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader) - val fetch = eachQuery(post) _ -// logger.info(jsonQuery) - - Try { - val future = jsonQuery match { - case JsArray(arr) => Future.traverse(arr.map(requestParser.toQuery(_)))(fetch).map(JsArray) - case obj@JsObject(_) => fetch(requestParser.toQuery(obj)) - case _ => throw BadQueryException("Cannot support") - } - - future map { json => jsonResponse(json, "result_size" -> calcSize(json).toString) } - - } recover { - case e: BadQueryException => - logger.error(s"$jsonQuery, $e", e) - badQueryExceptionResults(e) - case e: Exception => - logger.error(s"$jsonQuery, $e", e) - errorResults - } get - } - - @deprecated(message = "deprecated", since = "0.2") - private def getEdgesExcludedAsync(jsonQuery: JsValue) - (post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue): Future[Result] = { - - if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader) - - Try { - val q = requestParser.toQuery(jsonQuery) - val filterOutQuery = Query(q.vertices, Vector(q.steps.last)) - - val fetchFuture = s2.getEdges(q) - val excludeFuture = s2.getEdges(filterOutQuery) - - for { - queryResultLs <- fetchFuture - exclude <- excludeFuture - } yield { - val json = post(queryResultLs, exclude) - jsonResponse(json, "result_size" -> calcSize(json).toString) - } - } recover { - case e: BadQueryException => - logger.error(s"$jsonQuery, $e", e) - badQueryExceptionResults(e) - case e: Exception => - logger.error(s"$jsonQuery, $e", e) - errorResults - } get - } - - def getEdgesInner(jsonQuery: JsValue) = { - getEdgesAsync(jsonQuery)(PostProcess.toSimpleVertexArrJson) - } - - def getEdgesExcludedInner(jsValue: JsValue) = { - getEdgesExcludedAsync(jsValue)(PostProcess.toSimpleVertexArrJson) - } - - def getEdgesWithGrouping() = withHeaderAsync(jsonParser) { request => - getEdgesWithGroupingInner(request.body) - } - - def getEdgesWithGroupingInner(jsonQuery: JsValue) = { - getEdgesAsync(jsonQuery)(PostProcess.summarizeWithListFormatted) - } - - def getEdgesExcludedWithGrouping() = withHeaderAsync(jsonParser) { request => - getEdgesExcludedWithGroupingInner(request.body) - } - - def getEdgesExcludedWithGroupingInner(jsonQuery: JsValue) = { - getEdgesExcludedAsync(jsonQuery)(PostProcess.summarizeWithListExcludeFormatted) - } + def getEdgesExcludedWithGrouping() = withHeaderAsync(jsonParser)(delegate) - def getEdgesGroupedInner(jsonQuery: JsValue) = { - getEdgesAsync(jsonQuery)(PostProcess.summarizeWithList) - } + def checkEdges() = withHeaderAsync(jsonParser)(delegate) - @deprecated(message = "deprecated", since = "0.2") - def getEdgesGrouped() = withHeaderAsync(jsonParser) { request => - getEdgesGroupedInner(request.body) - } + def getEdgesGrouped() = withHeaderAsync(jsonParser)(delegate) - @deprecated(message = "deprecated", since = "0.2") - def getEdgesGroupedExcluded() = withHeaderAsync(jsonParser) { request => - getEdgesGroupedExcludedInner(request.body) - } + def getEdgesGroupedExcluded() = withHeaderAsync(jsonParser)(delegate) - @deprecated(message = "deprecated", since = "0.2") - def getEdgesGroupedExcludedInner(jsonQuery: JsValue): Future[Result] = { - if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader) + def getEdgesGroupedExcludedFormatted() = withHeaderAsync(jsonParser)(delegate) - Try { - val q = requestParser.toQuery(jsonQuery) - val filterOutQuery = Query(q.vertices, Vector(q.steps.last)) - - val fetchFuture = s2.getEdges(q) - val excludeFuture = s2.getEdges(filterOutQuery) - - for { - queryResultLs <- fetchFuture - exclude <- excludeFuture - } yield { - val json = PostProcess.summarizeWithListExclude(queryResultLs, exclude) - jsonResponse(json, "result_size" -> calcSize(json).toString) - } - } recover { - case e: BadQueryException => - logger.error(s"$jsonQuery, $e", e) - badQueryExceptionResults(e) - case e: Exception => - logger.error(s"$jsonQuery, $e", e) - errorResults - } get - } - - @deprecated(message = "deprecated", since = "0.2") - def getEdgesGroupedExcludedFormatted = withHeaderAsync(jsonParser) { request => - getEdgesGroupedExcludedFormattedInner(request.body) - } - - @deprecated(message = "deprecated", since = "0.2") - def getEdgesGroupedExcludedFormattedInner(jsonQuery: JsValue): Future[Result] = { - if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader) - - Try { - val q = requestParser.toQuery(jsonQuery) - val filterOutQuery = Query(q.vertices, Vector(q.steps.last)) - - val fetchFuture = s2.getEdges(q) - val excludeFuture = s2.getEdges(filterOutQuery) - - for { - queryResultLs <- fetchFuture - exclude <- excludeFuture - } yield { - val json = PostProcess.summarizeWithListExcludeFormatted(queryResultLs, exclude) - jsonResponse(json, "result_size" -> calcSize(json).toString) - } - } recover { - case e: BadQueryException => - logger.error(s"$jsonQuery, $e", e) - badQueryExceptionResults(e) - case e: Exception => - logger.error(s"$jsonQuery, $e", e) - errorResults - } get - } - - def getEdge(srcId: String, tgtId: String, labelName: String, direction: String) = Action.async { request => - if (!Config.IS_QUERY_SERVER) Future.successful(Unauthorized) - val params = Json.arr(Json.obj("label" -> labelName, "direction" -> direction, "from" -> srcId, "to" -> tgtId)) - checkEdgesInner(params) - } - - /** - * Vertex - */ - - def checkEdgesInner(jsValue: JsValue) = { - try { - val params = jsValue.as[List[JsValue]] - var isReverted = false - val labelWithDirs = scala.collection.mutable.HashSet[LabelWithDirection]() - val quads = for { - param <- params - labelName <- (param \ "label").asOpt[String] - direction <- GraphUtil.toDir((param \ "direction").asOpt[String].getOrElse("out")) - label <- Label.findByName(labelName) - srcId <- requestParser.jsValueToInnerVal((param \ "from").as[JsValue], label.srcColumnWithDir(direction.toInt).columnType, label.schemaVersion) - tgtId <- requestParser.jsValueToInnerVal((param \ "to").as[JsValue], label.tgtColumnWithDir(direction.toInt).columnType, label.schemaVersion) - } yield { - val labelWithDir = LabelWithDirection(label.id.get, direction) - labelWithDirs += labelWithDir - val (src, tgt, dir) = if (direction == 1) { - isReverted = true - (Vertex(VertexId(label.tgtColumnWithDir(direction.toInt).id.get, tgtId)), - Vertex(VertexId(label.srcColumnWithDir(direction.toInt).id.get, srcId)), 0) - } else { - (Vertex(VertexId(label.srcColumnWithDir(direction.toInt).id.get, srcId)), - Vertex(VertexId(label.tgtColumnWithDir(direction.toInt).id.get, tgtId)), 0) - } - - // logger.debug(s"SrcVertex: $src") - // logger.debug(s"TgtVertex: $tgt") - // logger.debug(s"direction: $dir") - (src, tgt, QueryParam(LabelWithDirection(label.id.get, dir))) - } - - s2.checkEdges(quads).map { case queryRequestWithResultLs => - val edgeJsons = for { - queryRequestWithResult <- queryRequestWithResultLs - (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get - edgeWithScore <- queryResult.edgeWithScoreLs - (edge, score) = EdgeWithScore.unapply(edgeWithScore).get - convertedEdge = if (isReverted) edge.duplicateEdge else edge - edgeJson = PostProcess.edgeToJson(convertedEdge, score, queryRequest.query, queryRequest.queryParam) - } yield Json.toJson(edgeJson) - - val json = Json.toJson(edgeJsons) - jsonResponse(json, "result_size" -> edgeJsons.size.toString) - } - } catch { - case e: Exception => - logger.error(s"$jsValue, $e", e) - errorResults + def getEdge(srcId: String, tgtId: String, labelName: String, direction: String) = + withHeaderAsync(jsonParser) { request => + val params = Json.arr(Json.obj("label" -> labelName, "direction" -> direction, "from" -> srcId, "to" -> tgtId)) + rest.checkEdges(params).map { js => + jsonResponse(js, "result_size" -> rest.calcSize(js).toString) + } recoverWith ApplicationController.requestFallback(request.body) } - } - - def checkEdges() = withHeaderAsync(jsonParser) { request => - if (!Config.IS_QUERY_SERVER) Future.successful(Unauthorized) - - checkEdgesInner(request.body) - } - - def getVertices() = withHeaderAsync(jsonParser) { request => - getVerticesInner(request.body) - } - - def getVerticesInner(jsValue: JsValue) = { - if (!Config.IS_QUERY_SERVER) Unauthorized.as(applicationJsonHeader) - - val jsonQuery = jsValue - val ts = System.currentTimeMillis() - val props = "{}" - - Try { - val vertices = jsonQuery.as[List[JsValue]].flatMap { js => - val serviceName = (js \ "serviceName").as[String] - val columnName = (js \ "columnName").as[String] - for (id <- (js \ "ids").asOpt[List[JsValue]].getOrElse(List.empty[JsValue])) yield { - Management.toVertex(ts, "insert", id.toString, serviceName, columnName, props) - } - } - s2.getVertices(vertices) map { vertices => - val json = PostProcess.verticesToJson(vertices) - jsonResponse(json, "result_size" -> calcSize(json).toString) - } - } recover { - case e: play.api.libs.json.JsResultException => - logger.error(s"$jsonQuery, $e", e) - badQueryExceptionResults(e) - case e: Exception => - logger.error(s"$jsonQuery, $e", e) - errorResults - } get - } + def getVertices() = withHeaderAsync(jsonParser)(delegate) } From 57d7b45619ea9d9834fbf7f191abe7c9929a38bd Mon Sep 17 00:00:00 2001 From: daewon Date: Wed, 6 Jan 2016 18:31:53 +0900 Subject: [PATCH 08/13] mend --- s2core/src/main/resources/reference.conf | 29 ++++++++ .../src/main/resources/reference.conf | 68 ------------------- 2 files changed, 29 insertions(+), 68 deletions(-) delete mode 100644 s2rest_netty/src/main/resources/reference.conf diff --git a/s2core/src/main/resources/reference.conf b/s2core/src/main/resources/reference.conf index b527f7f4..be1cb9ac 100644 --- a/s2core/src/main/resources/reference.conf +++ b/s2core/src/main/resources/reference.conf @@ -2,8 +2,34 @@ phase=dev host=localhost +# Kafka +kafka.metadata.broker.list=${host} +kafka.producer.pool.size=0 + +# Hbase +hbase.table.compression.algorithm="gz" hbase.zookeeper.quorum=${host} +# Local Queue Actor +local.queue.actor.max.queue.size=100000 +local.queue.actor.rate.limit=1000000 + +# Asynchbase +hbase.client.retries.number=100 +hbase.rpcs.buffered_flush_interval=100 +hbase.rpc.timeout=0 + +# local retry number +max.retry.number=100 +max.back.off=50 +delete.all.fetch.size=10000 +hbase.fail.prob=-1.0 + +# Future cache. +future.cache.max.size=100000 +future.cache.expire.after.write=10000 +future.cache.expire.after.access=5000 + # DB s2graph.models.table.name="models-dev" db.default.driver="com.mysql.jdbc.Driver" @@ -13,6 +39,9 @@ db.default.password="graph" cache.max.size=100000 +query.hard.limit=100000 +is.query.server=true + akka { loggers = ["akka.event.slf4j.Slf4jLogger"] loglevel = "DEBUG" diff --git a/s2rest_netty/src/main/resources/reference.conf b/s2rest_netty/src/main/resources/reference.conf deleted file mode 100644 index 7136b189..00000000 --- a/s2rest_netty/src/main/resources/reference.conf +++ /dev/null @@ -1,68 +0,0 @@ -# APP PHASE -phase=dev -host=localhost - -# DB -s2graph.models.table.name="models-dev" -hbase.zookeeper.quorum=${host} -db.default.url="jdbc:mysql://"${host}":3306/graph_dev" -# Query server -is.query.server=true -is.write.server=true -query.hard.limit=100000 - -# Local Cache -cache.ttl.seconds=60 -cache.max.size=100000 - -# HBASE -#hbase.client.operation.timeout=1000 -#async.hbase.client.flush.interval=100 -hbase.table.compression.algorithm="gz" - -# Asynchbase -hbase.client.retries.number=100 -hbase.rpcs.buffered_flush_interval=100 -hbase.rpc.timeout=0 -#hbase.nsre.high_watermark=1000000 -#hbase.timer.tick=5 -#hbase.timer.ticks_per_wheel=5 - -# Kafka -kafka.metadata.broker.list=${host} -kafka.producer.pool.size=0 - -# HTTP -parsers.text.maxLength=512K -parsers.json.maxLength=512K -trustxforwarded=false - -# Local Queue Actor -local.queue.actor.max.queue.size=100000 -local.queue.actor.rate.limit=1000000 - -# local retry number -max.retry.number=100 -max.back.off=50 -delete.all.fetch.size=10000 -hbase.fail.prob=-1.0 - -# max allowd edges for deleteAll is multiply of above two configuration. - -# set global obejct package, TODO: remove global -application.global=com.kakao.s2graph.rest.Global - -akka { - loggers = ["akka.event.slf4j.Slf4jLogger"] - loglevel = "DEBUG" -} - -# Future cache. -future.cache.max.size=100000 -future.cache.expire.after.write=10000 -future.cache.expire.after.access=5000 - - -# Counter -redis.instances = [${host}] - From f533ebf24b0c79132b1e914f5d349b01fb20ffd4 Mon Sep 17 00:00:00 2001 From: daewon Date: Wed, 6 Jan 2016 18:39:26 +0900 Subject: [PATCH 09/13] Revmoe unused config --- s2core/src/main/resources/reference.conf | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/s2core/src/main/resources/reference.conf b/s2core/src/main/resources/reference.conf index be1cb9ac..584c7d63 100644 --- a/s2core/src/main/resources/reference.conf +++ b/s2core/src/main/resources/reference.conf @@ -2,18 +2,10 @@ phase=dev host=localhost -# Kafka -kafka.metadata.broker.list=${host} -kafka.producer.pool.size=0 - # Hbase hbase.table.compression.algorithm="gz" hbase.zookeeper.quorum=${host} -# Local Queue Actor -local.queue.actor.max.queue.size=100000 -local.queue.actor.rate.limit=1000000 - # Asynchbase hbase.client.retries.number=100 hbase.rpcs.buffered_flush_interval=100 @@ -22,8 +14,6 @@ hbase.rpc.timeout=0 # local retry number max.retry.number=100 max.back.off=50 -delete.all.fetch.size=10000 -hbase.fail.prob=-1.0 # Future cache. future.cache.max.size=100000 @@ -37,11 +27,6 @@ db.default.url="jdbc:mysql://"${host}":3306/graph_dev" db.default.user="graph" db.default.password="graph" -cache.max.size=100000 - -query.hard.limit=100000 -is.query.server=true - akka { loggers = ["akka.event.slf4j.Slf4jLogger"] loglevel = "DEBUG" From c79fa0326febaa25616aed577e04750bd80308d9 Mon Sep 17 00:00:00 2001 From: daewon Date: Thu, 7 Jan 2016 11:11:30 +0900 Subject: [PATCH 10/13] S2GRAPH-9 add missing config --- s2core/src/main/resources/reference.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/s2core/src/main/resources/reference.conf b/s2core/src/main/resources/reference.conf index 584c7d63..cef02846 100644 --- a/s2core/src/main/resources/reference.conf +++ b/s2core/src/main/resources/reference.conf @@ -20,6 +20,10 @@ future.cache.max.size=100000 future.cache.expire.after.write=10000 future.cache.expire.after.access=5000 +# Local Cache +cache.ttl.seconds=60 +cache.max.size=100000 + # DB s2graph.models.table.name="models-dev" db.default.driver="com.mysql.jdbc.Driver" From 92112de53ab6eccb896c2d79cc7c06835618987e Mon Sep 17 00:00:00 2001 From: daewon Date: Thu, 7 Jan 2016 14:30:01 +0900 Subject: [PATCH 11/13] S2GRAPH-9 Refactoring RestCaller -> RestHandler --- .../{RestCaller.scala => RestHandler.scala} | 112 ++++++++++-------- s2rest_netty/src/main/scala/Server.scala | 91 ++++++-------- s2rest_play/app/Bootstrap.scala | 6 +- .../controllers/ExperimentController.scala | 12 +- .../app/controllers/QueryController.scala | 8 +- 5 files changed, 116 insertions(+), 113 deletions(-) rename s2core/src/main/scala/com/kakao/s2graph/core/rest/{RestCaller.scala => RestHandler.scala} (68%) diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestCaller.scala b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestHandler.scala similarity index 68% rename from s2core/src/main/scala/com/kakao/s2graph/core/rest/RestCaller.scala rename to s2core/src/main/scala/com/kakao/s2graph/core/rest/RestHandler.scala index bef1dece..96473145 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestCaller.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/rest/RestHandler.scala @@ -11,56 +11,52 @@ import play.api.libs.json._ import scala.concurrent.{ExecutionContext, Future} import scala.util.Try + +object RestHandler { + case class HandlerResult(body: Future[JsValue], headers: (String, String)*) +} + /** - * Public API only return Future.successful or Future.failed + * Public API, only return Future.successful or Future.failed * Don't throw exception */ -class RestCaller(graph: Graph)(implicit ec: ExecutionContext) { +class RestHandler(graph: Graph)(implicit ec: ExecutionContext) { + + import RestHandler._ + val s2Parser = new RequestParser(graph.config) /** * Public APIS */ - def experiment(contentsBody: JsValue, accessToken: String, experimentName: String, uuid: String): Future[(JsValue, String)] = { - try { - val bucketOpt = for { - service <- Service.findByAccessToken(accessToken) - experiment <- Experiment.findBy(service.id.get, experimentName) - bucket <- experiment.findBucket(uuid) - } yield bucket - - val bucket = bucketOpt.getOrElse(throw new RuntimeException("bucket is not found")) - if (bucket.isGraphQuery) buildRequestInner(contentsBody, bucket, uuid).map(_ -> bucket.impressionId) - else throw new RuntimeException("not supported yet") - } catch { - case e: Exception => Future.failed(e) - } - } - - def uriMatch(uri: String, jsQuery: JsValue): Future[JsValue] = { + def doPost(uri: String, jsQuery: JsValue): HandlerResult = { try { uri match { - case "/graphs/getEdges" => getEdgesAsync(jsQuery)(PostProcess.toSimpleVertexArrJson) - case "/graphs/getEdges/grouped" => getEdgesAsync(jsQuery)(PostProcess.summarizeWithListFormatted) - case "/graphs/getEdgesExcluded" => getEdgesExcludedAsync(jsQuery)(PostProcess.toSimpleVertexArrJson) - case "/graphs/getEdgesExcluded/grouped" => getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExcludeFormatted) + case "/graphs/getEdges" => HandlerResult(getEdgesAsync(jsQuery)(PostProcess.toSimpleVertexArrJson)) + case "/graphs/getEdges/grouped" => HandlerResult(getEdgesAsync(jsQuery)(PostProcess.summarizeWithListFormatted)) + case "/graphs/getEdgesExcluded" => HandlerResult(getEdgesExcludedAsync(jsQuery)(PostProcess.toSimpleVertexArrJson)) + case "/graphs/getEdgesExcluded/grouped" => HandlerResult(getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExcludeFormatted)) case "/graphs/checkEdges" => checkEdges(jsQuery) - case "/graphs/getEdgesGrouped" => getEdgesAsync(jsQuery)(PostProcess.summarizeWithList) - case "/graphs/getEdgesGroupedExcluded" => getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExclude) - case "/graphs/getEdgesGroupedExcludedFormatted" => getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExcludeFormatted) - case "/graphs/getVertices" => getVertices(jsQuery) + case "/graphs/getEdgesGrouped" => HandlerResult(getEdgesAsync(jsQuery)(PostProcess.summarizeWithList)) + case "/graphs/getEdgesGroupedExcluded" => HandlerResult(getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExclude)) + case "/graphs/getEdgesGroupedExcludedFormatted" => HandlerResult(getEdgesExcludedAsync(jsQuery)(PostProcess.summarizeWithListExcludeFormatted)) + case "/graphs/getVertices" => HandlerResult(getVertices(jsQuery)) + case uri if uri.startsWith("/graphs/experiment") => + val Array(accessToken, experimentName, uuid) = uri.split("/").takeRight(3) + experiment(jsQuery, accessToken, experimentName, uuid) case _ => throw new RuntimeException("route is not found") } } catch { - case e: Exception => Future.failed(e) + case e: Exception => HandlerResult(Future.failed(e)) } } - def checkEdges(jsValue: JsValue): Future[JsValue] = { + // TODO: Refactor to doGet + def checkEdges(jsValue: JsValue): HandlerResult = { try { val (quads, isReverted) = s2Parser.toCheckEdgeParam(jsValue) - graph.checkEdges(quads).map { case queryRequestWithResultLs => + HandlerResult(graph.checkEdges(quads).map { case queryRequestWithResultLs => val edgeJsons = for { queryRequestWithResult <- queryRequestWithResultLs (queryRequest, queryResult) = QueryRequestWithResult.unapply(queryRequestWithResult).get @@ -71,15 +67,51 @@ class RestCaller(graph: Graph)(implicit ec: ExecutionContext) { } yield Json.toJson(edgeJson) Json.toJson(edgeJsons) - } + }) } catch { - case e: Exception => Future.failed(e) + case e: Exception => HandlerResult(Future.failed(e)) } } + /** * Private APIS */ + private def experiment(contentsBody: JsValue, accessToken: String, experimentName: String, uuid: String): HandlerResult = { + try { + val bucketOpt = for { + service <- Service.findByAccessToken(accessToken) + experiment <- Experiment.findBy(service.id.get, experimentName) + bucket <- experiment.findBucket(uuid) + } yield bucket + + val bucket = bucketOpt.getOrElse(throw new RuntimeException("bucket is not found")) + if (bucket.isGraphQuery) { + val ret = buildRequestInner(contentsBody, bucket, uuid) + HandlerResult(ret.body, Experiment.impressionKey -> bucket.impressionId) + } + else throw new RuntimeException("not supported yet") + } catch { + case e: Exception => HandlerResult(Future.failed(e)) + } + } + + private def buildRequestInner(contentsBody: JsValue, bucket: Bucket, uuid: String): HandlerResult = { + if (bucket.isEmpty) HandlerResult(Future.successful(PostProcess.emptyResults)) + else { + val jsonBody = makeRequestJson(Option(contentsBody), bucket, uuid) + val url = new URL(bucket.apiPath) + val path = url.getPath() + + // dummy log for sampling + val experimentLog = s"POST $path took -1 ms 200 -1 $jsonBody" + + logger.info(experimentLog) + + doPost(path, jsonBody) + } + } + private def eachQuery(post: (Seq[QueryRequestWithResult], Seq[QueryRequestWithResult]) => JsValue)(q: Query): Future[JsValue] = { val filterOutQueryResultsLs = q.filterOutQuery match { case Some(filterOutQuery) => graph.getEdges(filterOutQuery) @@ -159,22 +191,6 @@ class RestCaller(graph: Graph)(implicit ec: ExecutionContext) { } get } - private def buildRequestInner(contentsBody: JsValue, bucket: Bucket, uuid: String): Future[JsValue] = { - if (bucket.isEmpty) Future.successful(PostProcess.emptyResults) - else { - val jsonBody = makeRequestJson(Option(contentsBody), bucket, uuid) - val url = new URL(bucket.apiPath) - val path = url.getPath() - - // dummy log for sampling - val experimentLog = s"POST $path took -1 ms 200 -1 $jsonBody" - - logger.info(experimentLog) - - uriMatch(path, jsonBody) - } - } - def calcSize(js: JsValue): Int = js match { case JsObject(obj) => (js \ "size").asOpt[Int].getOrElse(0) case JsArray(seq) => seq.map(js => (js \ "size").asOpt[Int].getOrElse(0)).sum diff --git a/s2rest_netty/src/main/scala/Server.scala b/s2rest_netty/src/main/scala/Server.scala index 10d12751..7290b8af 100644 --- a/s2rest_netty/src/main/scala/Server.scala +++ b/s2rest_netty/src/main/scala/Server.scala @@ -4,8 +4,8 @@ import java.util.concurrent.Executors import com.kakao.s2graph.core.GraphExceptions.BadQueryException import com.kakao.s2graph.core._ -import com.kakao.s2graph.core.mysqls.Experiment -import com.kakao.s2graph.core.rest.RestCaller +import com.kakao.s2graph.core.rest.RestHandler.HandlerResult +import com.kakao.s2graph.core.rest._ import com.kakao.s2graph.core.utils.Extensions._ import com.kakao.s2graph.core.utils.logger import com.typesafe.config.ConfigFactory @@ -21,22 +21,23 @@ import io.netty.handler.logging.{LogLevel, LoggingHandler} import io.netty.util.CharsetUtil import play.api.libs.json._ -import scala.concurrent.{ExecutionContext, Future} +import scala.collection.mutable +import scala.concurrent.ExecutionContext import scala.io.Source import scala.util.{Failure, Success, Try} -class S2RestHandler(s2rest: RestCaller)(implicit ec: ExecutionContext) extends SimpleChannelInboundHandler[FullHttpRequest] with JSONParser { +class S2RestHandler(s2rest: RestHandler)(implicit ec: ExecutionContext) extends SimpleChannelInboundHandler[FullHttpRequest] with JSONParser { val ApplicationJson = "application/json" val Ok = HttpResponseStatus.OK - val Close = ChannelFutureListener.CLOSE + val CloseOpt = Option(ChannelFutureListener.CLOSE) val BadRequest = HttpResponseStatus.BAD_REQUEST val BadGateway = HttpResponseStatus.BAD_GATEWAY val NotFound = HttpResponseStatus.NOT_FOUND val InternalServerError = HttpResponseStatus.INTERNAL_SERVER_ERROR def badRoute(ctx: ChannelHandlerContext) = - simpleResponse(ctx, BadGateway, byteBufOpt = None, channelFutureListenerOpt = Option(Close)) + simpleResponse(ctx, BadGateway, byteBufOpt = None, channelFutureListenerOpt = CloseOpt) def simpleResponse(ctx: ChannelHandlerContext, httpResponseStatus: HttpResponseStatus, @@ -59,49 +60,46 @@ class S2RestHandler(s2rest: RestCaller)(implicit ec: ExecutionContext) extends S } } - def toResponse(ctx: ChannelHandlerContext, req: FullHttpRequest, body: JsValue, future: Future[(JsValue, String)], startedAt: Long) = { + def toResponse(ctx: ChannelHandlerContext, req: FullHttpRequest, requestBody: JsValue, result: HandlerResult, startedAt: Long) = { + var closeOpt = CloseOpt + var headers = mutable.ArrayBuilder.make[(String, String)] - val defaultHeaders = List(Names.CONTENT_TYPE -> ApplicationJson) - // NOTE: logging size of result should move to s2core. - // logger.info(resJson.size.toString) - future onComplete { - case Success(resJsonWithImpId) => - val (resJson, impId) = resJsonWithImpId + headers += (Names.CONTENT_TYPE -> ApplicationJson) + result.headers.foreach(headers += _) - val duration = System.currentTimeMillis() - startedAt - val isKeepAlive = HttpHeaders.isKeepAlive(req) - val buf: ByteBuf = Unpooled.copiedBuffer(resJson.toString, CharsetUtil.UTF_8) - - - val (headerWithKeepAlive, listenerOpt) = - if (isKeepAlive) ((Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE) :: defaultHeaders) -> None - else defaultHeaders -> Option(Close) + if (HttpHeaders.isKeepAlive(req)) { + headers += (Names.CONNECTION -> HttpHeaders.Values.KEEP_ALIVE) + closeOpt = None + } - val headerWithImpKey = - if (impId.isEmpty) headerWithKeepAlive - else (Experiment.impressionKey, impId) :: headerWithKeepAlive + result.body onComplete { + case Success(json) => + val duration = System.currentTimeMillis() - startedAt - val log = s"${req.getMethod} ${req.getUri} took ${duration} ms 200 ${s2rest.calcSize(resJson)} ${body}" + val log = s"${req.getMethod} ${req.getUri} took ${duration} ms 200 ${s2rest.calcSize(json)} ${requestBody}" logger.info(log) - val responseHeaders = (Names.CONTENT_LENGTH -> buf.readableBytes().toString) :: headerWithImpKey - simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = listenerOpt, headers = responseHeaders) + val buf: ByteBuf = Unpooled.copiedBuffer(json.toString, CharsetUtil.UTF_8) + + headers += (Names.CONTENT_LENGTH -> buf.readableBytes().toString) + simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = closeOpt, headers = headers.result()) case Failure(ex) => ex match { case e: BadQueryException => - logger.error(s"{$body}, ${e.getMessage}", e) + logger.error(s"{$requestBody}, ${e.getMessage}", e) val buf: ByteBuf = Unpooled.copiedBuffer(PostProcess.badRequestResults(e).toString, CharsetUtil.UTF_8) - simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = Option(Close), headers = defaultHeaders) + simpleResponse(ctx, Ok, byteBufOpt = Option(buf), channelFutureListenerOpt = closeOpt, headers = headers.result()) case e: Exception => - logger.error(s"${body}, ${e.getMessage}", e) + logger.error(s"${requestBody}, ${e.getMessage}", e) val buf: ByteBuf = Unpooled.copiedBuffer(PostProcess.emptyResults.toString, CharsetUtil.UTF_8) - simpleResponse(ctx, InternalServerError, byteBufOpt = Option(buf), channelFutureListenerOpt = Option(Close), headers = defaultHeaders) + simpleResponse(ctx, InternalServerError, byteBufOpt = Option(buf), channelFutureListenerOpt = closeOpt, headers = headers.result()) } } } override def channelRead0(ctx: ChannelHandlerContext, req: FullHttpRequest): Unit = { val uri = req.getUri + val startedAt = System.currentTimeMillis() req.getMethod match { case HttpMethod.GET => @@ -109,18 +107,17 @@ class S2RestHandler(s2rest: RestCaller)(implicit ec: ExecutionContext) extends S case "/health_check.html" => if (NettyServer.isHealthy) { val healthCheckMsg = Unpooled.copiedBuffer(NettyServer.deployInfo, CharsetUtil.UTF_8) - simpleResponse(ctx, Ok, byteBufOpt = Option(healthCheckMsg), channelFutureListenerOpt = Option(Close)) + simpleResponse(ctx, Ok, byteBufOpt = Option(healthCheckMsg), channelFutureListenerOpt = CloseOpt) } else { - simpleResponse(ctx, NotFound, channelFutureListenerOpt = Option(Close)) + simpleResponse(ctx, NotFound, channelFutureListenerOpt = CloseOpt) } case s if s.startsWith("/graphs/getEdge/") => // src, tgt, label, dir val Array(srcId, tgtId, labelName, direction) = s.split("/").takeRight(4) val params = Json.arr(Json.obj("label" -> labelName, "direction" -> direction, "from" -> srcId, "to" -> tgtId)) - val startedAt = System.currentTimeMillis() - val future = s2rest.checkEdges(params) - toResponse(ctx, req, params, future.map(_ -> ""), startedAt) + val result = s2rest.checkEdges(params) + toResponse(ctx, req, params, result, startedAt) case _ => badRoute(ctx) } @@ -129,35 +126,25 @@ class S2RestHandler(s2rest: RestCaller)(implicit ec: ExecutionContext) extends S val newHealthCheck = uri.split("/").last.toBoolean NettyServer.isHealthy = newHealthCheck val newHealthCheckMsg = Unpooled.copiedBuffer(NettyServer.isHealthy.toString, CharsetUtil.UTF_8) - simpleResponse(ctx, Ok, byteBufOpt = Option(newHealthCheckMsg), channelFutureListenerOpt = Option(Close)) + simpleResponse(ctx, Ok, byteBufOpt = Option(newHealthCheckMsg), channelFutureListenerOpt = CloseOpt) } else badRoute(ctx) case HttpMethod.POST => val jsonString = req.content.toString(CharsetUtil.UTF_8) val jsQuery = Json.parse(jsonString) - //TODO: result_size - val startedAt = System.currentTimeMillis() - - val future = - if (uri.startsWith("/graphs/experiment")) { - val Array(accessToken, experimentName, uuid) = uri.split("/").takeRight(3) - s2rest.experiment(jsQuery, accessToken, experimentName, uuid) - } else { - s2rest.uriMatch(uri, jsQuery).map(_ -> "") - } - - toResponse(ctx, req, jsQuery, future, startedAt) + val result = s2rest.doPost(uri, jsQuery) + toResponse(ctx, req, jsQuery, result, startedAt) case _ => - simpleResponse(ctx, BadRequest, byteBufOpt = None, channelFutureListenerOpt = Option(Close)) + simpleResponse(ctx, BadRequest, byteBufOpt = None, channelFutureListenerOpt = CloseOpt) } } override def exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { cause.printStackTrace() logger.error(s"exception on query.", cause) - simpleResponse(ctx, BadRequest, byteBufOpt = None, channelFutureListenerOpt = Option(Close)) + simpleResponse(ctx, BadRequest, byteBufOpt = None, channelFutureListenerOpt = CloseOpt) } } @@ -174,7 +161,7 @@ object NettyServer extends App { // init s2graph with config val s2graph = new Graph(config)(ec) - val rest = new RestCaller(s2graph)(ec) + val rest = new RestHandler(s2graph)(ec) val deployInfo = Try(Source.fromFile("./release_info").mkString("")).recover { case _ => "release info not found\n" }.get var isHealthy = config.getBooleanWithFallback("app.health.on", true) diff --git a/s2rest_play/app/Bootstrap.scala b/s2rest_play/app/Bootstrap.scala index bc883f83..79bef4d8 100644 --- a/s2rest_play/app/Bootstrap.scala +++ b/s2rest_play/app/Bootstrap.scala @@ -3,7 +3,7 @@ package com.kakao.s2graph.rest import java.util.concurrent.Executors import actors.QueueActor -import com.kakao.s2graph.core.rest.{RestCaller, RequestParser} +import com.kakao.s2graph.core.rest._ import com.kakao.s2graph.core.utils.logger import com.kakao.s2graph.core.{ExceptionHandler, Graph} import config.Config @@ -19,7 +19,7 @@ import scala.util.Try object Global extends WithFilters(new GzipFilter()) { var s2graph: Graph = _ var s2parser: RequestParser = _ - var s2rest: RestCaller = _ + var s2rest: RestHandler = _ // Application entry point override def onStart(app: Application) { @@ -34,7 +34,7 @@ object Global extends WithFilters(new GzipFilter()) { // init s2graph with config s2graph = new Graph(config)(ec) s2parser = new RequestParser(s2graph.config) // merged config - s2rest = new RestCaller(s2graph)(ec) + s2rest = new RestHandler(s2graph)(ec) QueueActor.init(s2graph) diff --git a/s2rest_play/app/controllers/ExperimentController.scala b/s2rest_play/app/controllers/ExperimentController.scala index a4b5a17b..40f67d15 100644 --- a/s2rest_play/app/controllers/ExperimentController.scala +++ b/s2rest_play/app/controllers/ExperimentController.scala @@ -1,23 +1,23 @@ package controllers -import com.kakao.s2graph.core.mysqls.Experiment -import com.kakao.s2graph.core.rest.RestCaller +import com.kakao.s2graph.core.rest.RestHandler import play.api.mvc._ import scala.concurrent.ExecutionContext.Implicits.global object ExperimentController extends Controller { - private val rest: RestCaller = com.kakao.s2graph.rest.Global.s2rest + private val rest: RestHandler = com.kakao.s2graph.rest.Global.s2rest import ApplicationController._ def experiment(accessToken: String, experimentName: String, uuid: String) = withHeaderAsync(parse.anyContent) { request => val body = request.body.asJson.get + val res = rest.doPost(request.uri, body) - val res = rest.experiment(body, accessToken, experimentName, uuid) - res.map { case (js, impId) => - jsonResponse(js, Experiment.impressionKey -> impId, "result_size" -> rest.calcSize(js).toString) + res.body.map { case js => + val headers = res.headers :+ ("result_size" -> rest.calcSize(js).toString) + jsonResponse(js, headers: _*) } recoverWith ApplicationController.requestFallback(body) } } diff --git a/s2rest_play/app/controllers/QueryController.scala b/s2rest_play/app/controllers/QueryController.scala index 218d76e9..e8362590 100644 --- a/s2rest_play/app/controllers/QueryController.scala +++ b/s2rest_play/app/controllers/QueryController.scala @@ -2,7 +2,7 @@ package controllers import com.kakao.s2graph.core._ -import com.kakao.s2graph.core.rest.RestCaller +import com.kakao.s2graph.core.rest.RestHandler import play.api.libs.json.{JsValue, Json} import play.api.mvc.{Controller, Request} @@ -13,10 +13,10 @@ object QueryController extends Controller with JSONParser { import ApplicationController._ import play.api.libs.concurrent.Execution.Implicits.defaultContext - private val rest: RestCaller = com.kakao.s2graph.rest.Global.s2rest + private val rest: RestHandler = com.kakao.s2graph.rest.Global.s2rest def delegate(request: Request[JsValue]) = - rest.uriMatch(request.uri, request.body).map { js => + rest.doPost(request.uri, request.body).body.map { js => jsonResponse(js, "result_size" -> rest.calcSize(js).toString) } recoverWith ApplicationController.requestFallback(request.body) @@ -39,7 +39,7 @@ object QueryController extends Controller with JSONParser { def getEdge(srcId: String, tgtId: String, labelName: String, direction: String) = withHeaderAsync(jsonParser) { request => val params = Json.arr(Json.obj("label" -> labelName, "direction" -> direction, "from" -> srcId, "to" -> tgtId)) - rest.checkEdges(params).map { js => + rest.checkEdges(params).body.map { js => jsonResponse(js, "result_size" -> rest.calcSize(js).toString) } recoverWith ApplicationController.requestFallback(request.body) } From 64a167e4e3e9e62521d5001c4e4aee829173c905 Mon Sep 17 00:00:00 2001 From: daewon Date: Thu, 14 Jan 2016 19:02:32 +0900 Subject: [PATCH 12/13] Resolv conflict --- .../main/scala/com/kakao/s2graph/core/PostProcess.scala | 8 -------- 1 file changed, 8 deletions(-) diff --git a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala index 3a2420a1..56d8591c 100644 --- a/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala +++ b/s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala @@ -323,17 +323,9 @@ object PostProcess extends JSONParser { to <- innerValToJsValue(edge.tgtVertex.id.innerId, tgtColumn.columnType) } yield { val targetColumns = if (q.selectColumnsSet.isEmpty) reservedColumns else (reservedColumns & q.selectColumnsSet) + "props" -<<<<<<< HEAD:s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala - - val _propsMap = queryParam.label.metaPropsDefaultMapInner ++ propsToJson(edge, q, queryParam) - val propsMap = if (q.selectColumnsSet.nonEmpty) _propsMap.filterKeys(q.selectColumnsSet) else _propsMap - -======= - val _propsMap = queryParam.label.metaPropsDefaultMapInner ++ propsToJson(edge, q, queryParam) val propsMap = if (q.selectColumnsSet.nonEmpty) _propsMap.filterKeys(q.selectColumnsSet) else _propsMap ->>>>>>> 90777fd... S2GRAPH-7 Move PostProcess.scala to s2core from root project:s2core/src/main/scala/com/kakao/s2graph/core/PostProcess.scala val kvMap = targetColumns.foldLeft(Map.empty[String, JsValue]) { (map, column) => val jsValue = column match { case "cacheRemain" => JsNumber(queryParam.cacheTTLInMillis - (System.currentTimeMillis() - queryParam.timestamp)) From 11e80432e8c2c980bc49ea0b29ea52f2234aa81b Mon Sep 17 00:00:00 2001 From: daewon Date: Fri, 15 Jan 2016 10:47:53 +0900 Subject: [PATCH 13/13] Call initData() for initialize Model --- .../scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala index 393d8b1f..baf0743d 100644 --- a/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala +++ b/s2core/src/test/scala/com/kakao/s2graph/core/parsers/WhereParserTest.scala @@ -7,6 +7,8 @@ import org.scalatest.{FunSuite, Matchers} import play.api.libs.json.Json class WhereParserTest extends FunSuite with Matchers with TestCommonWithModels { + initTests() + // dummy data for dummy edge import HBaseType.{VERSION1, VERSION2}