diff --git a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala index 1f157d2..708d084 100644 --- a/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala +++ b/src/it/scala/de/upb/cs/swt/delphi/webapi/SearchQueryTest.scala @@ -16,17 +16,24 @@ package de.upb.cs.swt.delphi.webapi -import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchQuery} +import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchError, SearchQuery} import org.scalatest.{FlatSpec, Matchers} -import scala.util.Success +import scala.util.Failure class SearchQueryTest extends FlatSpec with Matchers { - "Search query" should "check for fields" in { + "Search query" should "fail on large request limit" in { val configuration = new Configuration() val q = new SearchQuery(configuration, new FeatureQuery(configuration)) - - val response = q.search(QueryRequest("[if_icmpeq (opcode:159)]>1")) - response shouldBe a [Success[_]] + val size = 20000 + val response = q.search(QueryRequest("[dstore_1 (opcode:72)]<1", Some(size))) + response match { + case Failure(exception) => { + exception shouldBe a[SearchError] + } + case _ => { + fail("Limit exceeded should fail") + } + } } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/DelphiRoutes.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/DelphiRoutes.scala index eab3de9..4fcfd8b 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/DelphiRoutes.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/DelphiRoutes.scala @@ -27,7 +27,7 @@ import de.upb.cs.swt.delphi.webapi.IpLogActor._ import de.upb.cs.swt.delphi.webapi.StatisticsJson._ import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactJson._ import de.upb.cs.swt.delphi.webapi.search.QueryRequestJson._ -import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchQuery} +import de.upb.cs.swt.delphi.webapi.search.{QueryRequest, SearchError, SearchQuery} import spray.json._ import scala.concurrent.duration._ @@ -118,7 +118,16 @@ class DelphiRoutes(requestLimiter: RequestLimitScheduler) extends JsonSupport wi complete( new SearchQuery(configuration, featureExtractor).search(input) match { case Success(result) => prettyPrint(pretty, result.toJson) - case Failure(e) => e.getMessage + case Failure(e) => { + e match { + case se: SearchError => { + se.toJson + } + case _ => { + new SearchError("Search query failed").toJson + } + } + } } ) } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequest.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequest.scala index 3479577..ffd57df 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequest.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequest.scala @@ -1,3 +1,18 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package de.upb.cs.swt.delphi.webapi.search -case class QueryRequest (query : String, limit : Option[Int] = Some(50)) +case class QueryRequest (query : String, limit : Option[Int] = Some(defaultFetchSize)) diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequestJson.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequestJson.scala index 6ae239e..09699a6 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequestJson.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/QueryRequestJson.scala @@ -1,3 +1,18 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package de.upb.cs.swt.delphi.webapi.search import spray.json.DefaultJsonProtocol diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala index 80c11b1..3caa421 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/SearchQuery.scala @@ -18,18 +18,18 @@ package de.upb.cs.swt.delphi.webapi.search import com.sksamuel.elastic4s.http.ElasticDsl._ import com.sksamuel.elastic4s.http.search.SearchHits -import com.sksamuel.elastic4s.http.{ElasticClient, RequestSuccess} +import com.sksamuel.elastic4s.http.{ElasticClient, RequestFailure, RequestSuccess} import com.sksamuel.elastic4s.searches.queries.{NoopQuery, Query} -import de.upb.cs.swt.delphi.webapi.{Configuration, FeatureQuery} -import de.upb.cs.swt.delphi.webapi.artifacts.ArtifactTransformer +import de.upb.cs.swt.delphi.webapi.artifacts.{Artifact, ArtifactTransformer} import de.upb.cs.swt.delphi.webapi.search.querylanguage._ +import de.upb.cs.swt.delphi.webapi.{Configuration, FeatureQuery} import scala.util.{Failure, Success, Try} class SearchQuery(configuration: Configuration, featureExtractor: FeatureQuery) { private val client = ElasticClient(configuration.elasticsearchClientUri) - private def checkAndExecuteParsedQuery(ast: CombinatorialExpr, limit : Int): Try[SearchHits] = { + private def checkAndExecuteParsedQuery(ast: CombinatorialExpr, limit: Int): Try[SearchHits] = { val fields = collectFieldNames(ast) if (fields.diff(featureExtractor.featureList.toSeq).size > 0) return Failure(new IllegalArgumentException("Unknown field name used.")) @@ -109,16 +109,42 @@ class SearchQuery(configuration: Configuration, featureExtractor: FeatureQuery) } } - def search(query: QueryRequest) = { - val parserResult = new Syntax(query.query).QueryRule.run() - parserResult match { - case Failure(e) => Failure(e) - case Success(ast) => { - checkAndExecuteParsedQuery(ast, query.limit.getOrElse(50)) match { - case Failure(e) => Failure(e) - case Success(hits) => Success(ArtifactTransformer.transformResults(hits)) + def checkValidSize: Option[Int] = { + import elastic4s.extns._ + import elastic4s.extns.ElasticDslExtn._ + val params = Map("include_defaults" -> true) + val query = SettingsRequest("delphi", params) + val res = client.execute { + query + }.await + res match { + case RequestSuccess(_, b, _, _) => { + maxResultSize(b, configuration) + } + case RequestFailure(_, _, _, _) => { + None + } + } + } + + def search(query: QueryRequest): Try[Array[Artifact]] = { + lazy val size = checkValidSize + val validSize = size.exists(query.limit.getOrElse(defaultFetchSize) <= _) + if (validSize) { + val parserResult = new Syntax(query.query).QueryRule.run() + parserResult match { + case Failure(e) => Failure(e) + case Success(ast) => { + checkAndExecuteParsedQuery(ast, query.limit.getOrElse(defaultFetchSize)) match { + case Failure(e) => Failure(e) + case Success(hits) => Success(ArtifactTransformer.transformResults(hits)) + } } } } + else { + val errorMsg = new SearchError(s"Query limit exceeded default limit: ${query.limit}>${size}") + Failure(errorMsg) + } } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/elastic4s/extns/package.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/elastic4s/extns/package.scala new file mode 100644 index 0000000..fe5332a --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/elastic4s/extns/package.scala @@ -0,0 +1,66 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package de.upb.cs.swt.delphi.webapi.search.elastic4s + +import com.sksamuel.elastic4s.http.settings.IndexSettingsResponse +import com.sksamuel.elastic4s.http.{ElasticRequest, Handler} +import com.sksamuel.elastic4s.json.JacksonSupport +import de.upb.cs.swt.delphi.webapi.Configuration + +package object extns { + + case class SettingsRequest(index: String, params: Map[String, Any]) + + trait RichSettingsHandler { + + implicit object RichGetSettings extends Handler[SettingsRequest, IndexSettingsResponse] { + + override def build(request: SettingsRequest): ElasticRequest = { + val endpoint = "/" + request.index + "/_settings" + val req = ElasticRequest("GET", endpoint, request.params) + req + } + } + + } + + def maxResultSize(res: Option[String], config: Configuration): Option[Int] = { + res match { + case Some(j) => { + val custom = s"/${config.esIndex}/settings/index" + val default = s"/${config.esIndex}/defaults/index" + val target = "max_result_window" + val node = JacksonSupport.mapper.readTree(j) + val size = if (node.at(custom).has(target)) { + Some(node.at(custom + "/" + target).asInt()) + } + else { + Some(node.at(default + "/" + target).asInt()) + } + size + } + case None => { + None + } + } + } + + + trait ElasticDslExtn extends RichSettingsHandler + + object ElasticDslExtn extends ElasticDslExtn + +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/webapi/search/package.scala b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/package.scala new file mode 100644 index 0000000..e0b8189 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/webapi/search/package.scala @@ -0,0 +1,33 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.webapi + +import spray.json._ + +package object search { + + val defaultFetchSize = 50 + + class SearchError(msg: String) extends RuntimeException(msg) with JsonSupport + + implicit val searchErrorWriter = new JsonWriter[SearchError] { + override def write(obj: SearchError): JsValue = { + JsObject("msg" -> JsString(obj.getMessage)) + } + } + +}