Permalink
Browse files

Enhanced unit tests for the AttributeFacetHandler

  • Loading branch information...
2 parents 19af807 + 757a440 commit 8c9bd1291b0d624b0f78ca7a5540076ac5ae047c Volodymyr Zhabiuk committed with vzhabiuk Jan 11, 2012
Showing with 50,445 additions and 24,672 deletions.
  1. +4 −0 .classpath
  2. +2 −2 clients/java/src/main/java/com/sensei/search/client/json/SenseiServiceProxy.java
  3. +1 −1 clients/java/src/test/java/com/sensei/search/client/json/JavaClientIntegrationTest.java
  4. +2 −2 clients/python/sensei/bql_parser.py
  5. +53 −16 clients/python/sensei/sensei_client.py
  6. +1 −0 clients/python/sensei/sensei_components.py
  7. +2 −2 clients/python/test/test_json_api.py
  8. +1 −1 docs/src/docbkx/chap-faq.xml
  9. +4 −2 example/cars/conf/sensei.properties
  10. +11,678 −11,678 example/cars/data/cars.json
  11. +3 −1 example/tweets/conf/sensei.properties
  12. +1 −1 example/tweets/src/main/java/com/senseidb/example/tweets/gateway/TwitterSampleStreamer.java
  13. +2 −1 pom.xml
  14. +27 −8 {src/main/assembly → resources}/release.xml
  15. +71 −0 sensei-core/.classpath
  16. +13 −0 sensei-core/.project
  17. +70 −33 sensei-core/pom.xml
  18. +1,545 −0 sensei-core/src/main/antlr3/com/sensei/bql/parsers/BQL.g
  19. +49 −0 sensei-core/src/main/java/com/sensei/bql/parsers/AbstractCompiler.java
  20. +60 −0 sensei-core/src/main/java/com/sensei/bql/parsers/BQLCompiler.java
  21. +1 −0 sensei-core/src/main/java/com/sensei/conf/SenseiConfParams.java
  22. +39 −8 sensei-core/src/main/java/com/sensei/conf/SenseiServerBuilder.java
  23. +1 −1 sensei-core/src/main/java/com/sensei/examples/SchemaIndexingExample.java
  24. +2 −38 sensei-core/src/main/java/com/sensei/facet/attribute/FacetPredicate.java
  25. +12 −5 sensei-core/src/main/java/com/sensei/facet/attribute/PredicateFacetFilter.java
  26. +62 −31 sensei-core/src/main/java/com/sensei/facet/attribute/RangePredicate.java
  27. +86 −0 sensei-core/src/main/java/com/sensei/search/client/servlet/AbstractSenseiClientServlet.java
  28. +51 −13 sensei-core/src/main/java/com/sensei/search/client/servlet/DefaultSenseiJSONServlet.java
  29. +27 −11 sensei-core/src/main/java/com/sensei/search/nodes/SenseiBroker.java
  30. +4 −3 sensei-core/src/main/java/com/sensei/search/nodes/SenseiHourglassFactory.java
  31. +4 −2 sensei-core/src/main/java/com/sensei/search/nodes/SenseiPairFactory.java
  32. +0 −3 sensei-core/src/main/java/com/sensei/search/nodes/SenseiServer.java
  33. +1 −1 sensei-core/src/main/java/com/sensei/search/nodes/SenseiSysBroker.java
  34. +4 −1 sensei-core/src/main/java/com/sensei/search/nodes/SenseiZoieFactory.java
  35. +8 −3 sensei-core/src/main/java/com/sensei/search/nodes/SenseiZoieSystemFactory.java
  36. +4 −1 sensei-core/src/main/java/com/sensei/search/nodes/impl/DemoZoieSystemFactory.java
  37. +1 −1 sensei-core/src/main/java/com/sensei/search/query/QueryStringQueryConstructor.java
  38. +11 −0 sensei-core/src/main/java/com/sensei/search/req/SenseiHit.java
  39. +24 −0 sensei-core/src/main/java/com/sensei/search/req/SenseiRequest.java
  40. +0 −252 sensei-core/src/main/java/com/sensei/search/req/protobuf/ProtoConvertUtil.java
  41. +0 −138 sensei-core/src/main/java/com/sensei/search/req/protobuf/SenseiGenericBPOConverter.java
  42. +0 −503 sensei-core/src/main/java/com/sensei/search/req/protobuf/SenseiGenericRequestBPO.java
  43. +0 −503 sensei-core/src/main/java/com/sensei/search/req/protobuf/SenseiGenericResultBPO.java
  44. +0 −54 sensei-core/src/main/java/com/sensei/search/req/protobuf/SenseiReqProtoSerializer.java
  45. +0 −400 sensei-core/src/main/java/com/sensei/search/req/protobuf/SenseiRequestBPO.java
  46. +0 −88 sensei-core/src/main/java/com/sensei/search/req/protobuf/SenseiRequestBPOConverter.java
  47. +0 −400 sensei-core/src/main/java/com/sensei/search/req/protobuf/SenseiResultBPO.java
  48. +0 −55 sensei-core/src/main/java/com/sensei/search/req/protobuf/SenseiSysReqProtoSerializer.java
  49. +0 −400 sensei-core/src/main/java/com/sensei/search/req/protobuf/SenseiSysRequestBPO.java
  50. +0 −89 sensei-core/src/main/java/com/sensei/search/req/protobuf/SenseiSysRequestBPOConverter.java
  51. +0 −400 sensei-core/src/main/java/com/sensei/search/req/protobuf/SenseiSysResultBPO.java
  52. +0 −4 sensei-core/src/main/java/com/sensei/search/svc/impl/CoreSenseiServiceImpl.java
  53. +0 −4 sensei-core/src/main/java/com/sensei/search/svc/impl/SysSenseiCoreServiceImpl.java
  54. +62 −0 sensei-core/src/main/java/com/sensei/search/util/HDFSIndexCopier.java
  55. +0 −1 sensei-core/src/main/java/com/sensei/search/util/RequestConverter.java
  56. +20 −3 sensei-core/src/main/java/com/sensei/search/util/RequestConverter2.java
  57. +104 −0 sensei-core/src/main/webapp/css/codemirror.css
  58. +305 −134 sensei-core/src/main/webapp/index.html
  59. +23 −0 sensei-core/src/main/webapp/js/CODEMIRROR_LICENSE
  60. +2,764 −0 sensei-core/src/main/webapp/js/codemirror.js
  61. +351 −182 sensei-core/src/main/webapp/js/console-app.js
  62. +29 −0 sensei-core/src/main/webapp/js/keymap/emacs.js
  63. +347 −0 sensei-core/src/main/webapp/js/keymap/vim.js
  64. +188 −0 sensei-core/src/main/webapp/js/mode/mysql/mysql.js
  65. +24 −0 sensei-core/src/main/webapp/js/mode/ruby/LICENSE
  66. +195 −0 sensei-core/src/main/webapp/js/mode/ruby/ruby.js
  67. +232 −0 sensei-core/src/test/java/com/sensei/bql/parsers/JsonComparator.java
  68. +1,020 −0 sensei-core/src/test/java/com/sensei/bql/parsers/TestBQL.java
  69. +706 −0 sensei-core/src/test/java/com/sensei/bql/parsers/TestErrorHandling.java
  70. +84 −0 sensei-core/src/test/java/com/sensei/facet/attributes/DataConverter.java
  71. +7 −11 sensei-core/src/test/java/com/sensei/test/SenseiStarter.java
  72. +9 −2 sensei-core/src/test/java/com/sensei/test/TestSensei.java
  73. +22 −21 sensei-core/src/test/java/com/sensei/test/TestSenseiAttributesHandler.java
  74. +0 −188 sensei-core/src/test/java/com/sensei/test/TestSerialization.java
  75. +2 −1 sensei-core/src/test/java/com/sensei/test/ZoieTestFactory.java
  76. +7 −0 sensei-core/src/test/json/sensei-request.json
  77. +30,000 −0 sensei-core/src/test/resources/data/test_data.json
  78. 0 sensei-core/src/test/resources/{conf → test-conf}/bobo.spring
  79. +2 −2 sensei-core/src/test/resources/{conf → test-conf}/node1/schema.xml
  80. +2 −2 sensei-core/src/test/resources/{conf → test-conf}/node1/sensei.properties
  81. +3 −2 sensei-core/src/test/resources/{conf → test-conf}/node2/schema.xml
  82. +3 −3 sensei-core/src/test/resources/{conf → test-conf}/node2/sensei.properties
  83. 0 sensei-core/src/test/resources/{conf → test-conf}/sensei-test.spring
  84. 0 sensei-core/src/test/resources/{conf → test-conf}/zoie.spring
  85. +1 −1 ...util → sensei-gateways/src/main/java/com/senseidb/sensei/gateway/kafka}/AvroSerializerHelper.java
  86. +2 −2 sensei-hadoop-indexing/src/main/java/org/apache/lucene/store/TestRAMDirectorySerializer.java
  87. +0 −102 src/main/webapp/admin.html
  88. +0 −107 src/main/webapp/css/admin.css
  89. +0 −112 src/main/webapp/css/humanmsg.css
  90. +0 −110 src/main/webapp/css/main.css
  91. +0 −488 src/main/webapp/demo.html
  92. +0 −109 src/main/webapp/index.html
  93. +0 −811 src/main/webapp/js/admin.js
  94. +0 −1,012 src/main/webapp/js/backbone.js
  95. +0 −283 src/main/webapp/js/console.js
  96. +0 −86 src/main/webapp/js/humanmsg.js
  97. +0 −4,544 src/main/webapp/js/jquery.jstree.js
  98. +0 −367 src/main/webapp/js/mustache.js
  99. +0 −530 src/main/webapp/js/sensei-client.js
  100. BIN src/main/webapp/js/themes/apple/bg.jpg
  101. BIN src/main/webapp/js/themes/apple/d.png
  102. BIN src/main/webapp/js/themes/apple/dot_for_ie.gif
  103. +0 −61 src/main/webapp/js/themes/apple/style.css
  104. BIN src/main/webapp/js/themes/apple/throbber.gif
  105. BIN src/main/webapp/js/themes/classic/d.gif
  106. BIN src/main/webapp/js/themes/classic/d.png
  107. BIN src/main/webapp/js/themes/classic/dot_for_ie.gif
  108. +0 −77 src/main/webapp/js/themes/classic/style.css
  109. BIN src/main/webapp/js/themes/classic/throbber.gif
  110. BIN src/main/webapp/js/themes/default-rtl/d.gif
  111. BIN src/main/webapp/js/themes/default-rtl/d.png
  112. BIN src/main/webapp/js/themes/default-rtl/dots.gif
  113. +0 −84 src/main/webapp/js/themes/default-rtl/style.css
  114. BIN src/main/webapp/js/themes/default-rtl/throbber.gif
  115. BIN src/main/webapp/js/themes/default/d.gif
  116. BIN src/main/webapp/js/themes/default/d.png
  117. +0 −74 src/main/webapp/js/themes/default/style.css
  118. BIN src/main/webapp/js/themes/default/throbber.gif
View
@@ -5,6 +5,7 @@
<classpathentry kind="src" path="clients/java-client/src/test/java"/>
<classpathentry kind="src" path="example/hadoop-indexing/src/main/java"/>
<classpathentry kind="src" path="sensei-hadoop-indexing/src/main/java"/>
+ <classpathentry kind="src" path="sensei-core/target/generated-sources/antlr3"/>
<classpathentry kind="src" path="perf/src/main/java"/>
<classpathentry kind="src" path="sensei-core/src/test/java"/>
<classpathentry kind="lib" path="lib/commons-httpclient-3.1.jar"/>
@@ -13,6 +14,9 @@
<classpathentry kind="var" path="M2_REPO/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar"/>
<classpathentry kind="var" path="M2_REPO/org/apache/ant/ant/1.7.1/ant-1.7.1.jar"/>
<classpathentry kind="var" path="M2_REPO/org/apache/ant/ant-launcher/1.7.1/ant-launcher-1.7.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/antlr/antlr/3.4/antlr-3.4.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/antlr/antlr-runtime/3.4/antlr-runtime-3.4.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/antlr/antlr3-maven-plugin/3.4/antlr3-maven-plugin-3.4.jar"/>
<classpathentry kind="var" path="M2_REPO/aopalliance/aopalliance/1.0/aopalliance-1.0.jar"/>
<classpathentry kind="var" path="M2_REPO/asm/asm/3.2/asm-3.2.jar"/>
<classpathentry kind="var" path="M2_REPO/org/apache/avro/avro/1.4.0/avro-1.4.0.jar"/>
@@ -62,8 +62,8 @@ public SenseiResult sendSearchRequest( SenseiClientRequest request) {
}
}
- public Map<Long, JSONObject> sendGetRequest(List<Long> uids) throws IOException, JSONException {
- Map<Long, JSONObject> ret = new LinkedHashMap<Long, JSONObject>(uids.size());
+ public Map<Long, JSONObject> sendGetRequest(long... uids) throws IOException, JSONException {
+ Map<Long, JSONObject> ret = new LinkedHashMap<Long, JSONObject>(uids.length);
String response = sendPostRaw(getStoreGetUrl(), new JSONArray(uids).toString());
if (response == null || response.length() == 0) {
return ret;
@@ -385,7 +385,7 @@ public void testTemplateMapping() throws Exception
public void testGetStoreQuery() throws Exception
{
- Map<Long, JSONObject> ret = senseiServiceProxy.sendGetRequest(Arrays.asList(1L,2L, 3L, 5L));
+ Map<Long, JSONObject> ret = senseiServiceProxy.sendGetRequest(1L,2L, 3L, 5L);
assertEquals(4, ret.size());
assertEquals(Integer.valueOf(1), ret.get(1L).get("id"));
@@ -548,14 +548,14 @@ def between_predicate_action(self, s, loc, tok):
raise ParseSyntaxException(ParseException(s, loc, msg))
return {"or": [{"range":
{tok[0]:
- {"to": to_value,
+ {"to": from_value,
"include_upper": False
}
}
},
{"range":
{tok[0]:
- {"from": from_value,
+ {"from": to_value,
"include_lower": False
}
}
@@ -244,7 +244,9 @@ def doQuery(self, req, using_json=True):
time1 = datetime.now()
query_string = None
if using_json: # Use JSON format
- query_string = self.buildJsonString(req)
+ # query_string = self.buildJsonString(req)
+ bql = {"bql": req}
+ query_string = json.dumps(bql)
else:
query_string = SenseiClient.buildUrlString(req)
logger.debug(query_string)
@@ -307,6 +309,16 @@ def run_example(self):
res.display(columns, max_col_width=40)
def main(argv):
+
+ def help():
+ print """\
+help Show instructions
+select ... Execute a BQL statement
+desc | describe Describe current index schema
+get <uid_list> Retrieve documents based on UID list
+exit Exit
+"""
+
print "Welcome to Sensei Shell"
from optparse import OptionParser
usage = "usage: %prog [options]"
@@ -329,34 +341,57 @@ def main(argv):
if len(args) <= 1:
client = SenseiClient()
- print "using default host=localhost, port=8080"
+ print "Using default host=localhost, port=8080"
else:
host = args[0]
port = int(args[1])
print "Url specified, host: %s, port: %d" % (host,port)
client = SenseiClient(host, port, 'sensei')
+ print 'Enter "help" for instructions'
+
import readline
readline.parse_and_bind("tab: complete")
while 1:
try:
- stmt = raw_input('> ')
- if stmt == "exit":
- break
+ stmt = raw_input('bql> ')
- # if options.verbose:
- # test(stmt)
- if stmt == "sample request":
- client.run_example()
- continue
-
- req = client.compile(stmt)
- if req.stmt_type == "select":
- res = client.doQuery(req)
- res.display(columns=req.get_columns(), max_col_width=int(options.max_col_width))
- elif req.stmt_type == "desc":
+ words = re.split(r'[\s,]+', stmt.strip())
+
+ command = len(words) > 0 and words[0].lower() or None
+
+ if len(words) == 1 and command == "exit":
+ break
+ elif command == "get":
+ if len(words) > 1:
+ stmt = "select * where _uid in (%s)" % ','.join(words[1:])
+ command = "select"
+ else:
+ print "Bad GET command."
+ continue
+
+ if command == "select":
+ res = client.doQuery(stmt)
+ error = res.error
+ if error:
+ err_code = error.get("code")
+ err_msg = error.get("msg")
+ if err_code == 499:
+ err_match = re.match(r'^\[line:(\d+), col:(\d+)\].*', err_msg)
+ print '%s^' % (' ' * (5 + int(err_match.group(2))))
+ print err_msg
+ else:
+ print "Unknown error happened!"
+ else:
+ res.display(columns=["*"], max_col_width=int(options.max_col_width))
+ elif command in ["desc", "describe"]:
sysinfo = client.get_sysinfo()
sysinfo.display()
+ elif command == "help":
+ help()
+ else:
+ print "Unrecognized command."
+
except EOFError:
break
except ParseException as err:
@@ -367,6 +402,8 @@ def main(argv):
print " " * (err.loc + 2) + "^\n" + err.msg
except SenseiClientError as err:
print err
+ except Exception as err:
+ print err
if __name__ == "__main__":
main(sys.argv)
@@ -1002,6 +1002,7 @@ def __init__(self, json_data):
self.total_time = 0
self.numHits = json_data.get(PARAM_RESULT_NUMHITS, 0)
self.hits = json_data.get(PARAM_RESULT_HITS)
+ self.error = json_data.get("error")
map = json_data.get(PARAM_RESULT_FACETS)
self.facetMap = {}
if map:
@@ -1152,14 +1152,14 @@ def testBetweenPred2(self):
"range": {
"year": {
"include_upper": false,
- "to": 2001
+ "to": 2000
}
}
},
{
"range": {
"year": {
- "from": 2000,
+ "from": 2001,
"include_lower": false
}
}
@@ -36,7 +36,7 @@
xlink:href="http://www.elasticsearch.org/guide/reference/query-dsl/">ElasticSearch's
Query DSL</link>. ElasticSearch's Query DSL is very well
designed in exposing <link
- xlink:href="http://lucene.apache.org/java/3_4_0/api/core/org/apache/lucene/search/Query.html">Lucene's
+ xlink:href="http://lucene.apache.org/java/3_5_0/api/core/org/apache/lucene/search/Query.html">Lucene's
query</link> capabilities and features in an elegant way.
We didn't think it was necessary to create yet another API
that is completely different.</para>
@@ -28,7 +28,10 @@ sensei.index.directory = /tmp/sensei-example-cars
#sensei.index.batchDelay = 300000
#sensei.index.maxBatchSize = 10000
#sensei.index.realtime = true
-#sensei.index.freshness = 1000
+
+
+# indicator of freshness of data, in seconds, a number <=0 implies as fast as possible
+sensei.index.freshness = 5
# gateway parameters
@@ -55,4 +58,3 @@ sensei.broker.port = 8080
#sensei.search.cluster.network.max.conn.per.node = 5
#sensei.search.cluster.network.stale.timeout.mins = 10
#sensei.search.cluster.network.stale.cleanup.freq.mins = 10
-
Oops, something went wrong.

0 comments on commit 8c9bd12

Please sign in to comment.