Search tests

java-app/get_sdk.sh

@@ -1,15 +1,28 @@
+#!/bin/bash
+
 JAVA_SDK_VERSION=1.8.4
 JAVA_SDK_ZIP=appengine-java-sdk-${JAVA_SDK_VERSION}.zip
+SDK_URL="http://s3.amazonaws.com/appscale-build/${JAVA_SDK_ZIP}"
 
 cd ../../
 if [ -e ${HOME}/${JAVA_SDK_ZIP} ]; then
         cp ${HOME}/${JAVA_SDK_ZIP} .
 else
-        wget http://googleappengine.googlecode.com/files/${JAVA_SDK_ZIP}
+        if ! wget ${SDK_URL}; then
+                echo "Failed to retrieve SDK from ${SDK_URL}, exiting script"
+                cd -
+                exit 1
+        fi
 fi
 
-echo "Extracting ${JAVA_SDK_ZIP}"
-unzip -q ${JAVA_SDK_ZIP}
-rm ${JAVA_SDK_ZIP}
+echo "Extracting ${JAVA_SDK_ZIP} to ${PWD}"
+
+if ! unzip -q ${JAVA_SDK_ZIP}; then
+        echo "Failed to unzip SDK correctly, please see errors above"
+        rm -f ${JAVA_SDK_ZIP}
+        cd -
+        exit 1
+fi
 
+rm -f ${JAVA_SDK_ZIP}
 cd -

python27-app/module-main/app.yaml

@@ -83,6 +83,9 @@ handlers:
 - url: /_ah/xmpp/message/chat/?
   script: main.app
 
+- url: /python/search/(.*)
+  script: main.app
+
 - url: /.*
   script: main.app
 

python27-app/module-main/main.py

@@ -11,6 +11,7 @@
 from memcache import urls as memcache_urls
 from module_main import urls as modules_urls
 from ndb import urls as ndb_urls
+from search import urls as search_urls
 from secure_url import urls as secure_url_urls
 from taskqueue import urls as taskqueue_urls
 from urlfetch import urls as urlfetch_urls
@@ -33,5 +34,6 @@
   taskqueue_urls +
   urlfetch_urls +
   user_urls +
-  xmpp_urls
+  xmpp_urls +
+  search_urls
 )

python27-app/module-main/search.py

@@ -0,0 +1,149 @@
+import json
+import datetime
+import logging
+import wsgiref
+
+import webapp2
+
+from google.appengine.api.search import search
+from google.appengine.api.search.search import QueryOptions, ScoredDocument, Query
+from google.appengine.ext import webapp
+
+field_type_dict = {'text':search.TextField,
+                   'html':search.HtmlField,
+                   'atom':search.AtomField,
+                   'number':search.NumberField,
+                   'date':search.DateField}
+
+def get_field(field_name):
+  return field_type_dict[field_name]
+
+def render_doc(document):
+  """
+  Generates JSON-compatible object from object of type search.Document
+  All fields are expected to be TextField for now
+  :param document: document to render
+  :type document: search.Document
+  :return: dict {<field_name>: <string_repr_of_value>}
+  """
+  if not document:
+    return None
+  document_dict = {
+    field.name: unicode(field.value) for field in document.fields
+  }
+  # Pull over document id as it isn't included in fields.
+  document_dict['id'] = document.doc_id
+
+  if isinstance(document, ScoredDocument):
+    document_dict["_sort_scores"] = document.sort_scores
+  return document_dict
+
+
+def parse_doc(raw_document):
+  """
+  Builds object of type search.Document from dict
+  Only TextFields are supported for now
+  :param raw_document: {<field_name>: <text_field_value>}
+  :return: search.Document
+  """
+  fields = []
+  # make sure fields exists?
+  for f in raw_document.get('fields'):
+
+    field = get_field(f['type'])       # get class by simple name: text,atom
+    if f['type'] in ['text','html']:
+      # text/html has a lang field, lang should be a two character
+      # specifier as required by the sdk.
+      fields.append(field(f['name'],
+                          f['value'],
+                          f['lang'] if 'lang' in f else 'en'))
+    elif f['type'] == 'date':
+      # for date field we need to convert to a datetime
+      fields.append(field(f['name'],
+                    datetime.datetime.strptime(f['value'], '%Y-%m-%d')))
+    else:
+      # All other fields just have a name/value pair.
+      fields.append(field(f['name'],
+                          f['value']))
+
+  return search.Document(
+    doc_id=raw_document.get('id'),
+    fields=fields
+  )
+
+
+
+class PutDocumentsHandler(webapp2.RequestHandler):
+
+  def post(self):
+    payload = json.loads(self.request.body)
+    index = payload['index']
+    raw_documents = payload['documents']
+    documents = [parse_doc(raw_doc) for raw_doc in raw_documents]
+    put_results = search.Index(name=index).put(documents)
+    response = {'document_ids': [result.id for result in put_results]}
+    self.response.headers['Content-Type'] = 'application/json'
+    self.response.out.write(json.dumps(response))
+
+
+class GetDocumentHandler(webapp2.RequestHandler):
+
+  def post(self):
+    payload = json.loads(self.request.body)
+    index = payload['index']
+    doc_id = payload['id']
+    document = search.Index(name=index).get(doc_id)
+    response = {'document': render_doc(document)}
+    self.response.headers['Content-Type'] = 'application/json'
+    self.response.out.write(json.dumps(response))
+
+
+class GetDocumentsRangeHandler(webapp2.RequestHandler):
+
+  def post(self):
+    payload = json.loads(self.request.body)
+    index = payload['index']
+    start_id = payload['start_id']
+    limit = payload.get('limit')
+    index = search.Index(name=index)
+    documents = index.get_range(start_id=start_id, limit=limit)
+    response = {'documents': [render_doc(document) for document in documents]}
+    self.response.headers['Content-Type'] = 'application/json'
+    self.response.out.write(json.dumps(response))
+
+
+class SearchDocumentsHandler(webapp2.RequestHandler):
+
+  def post(self):
+    payload = json.loads(self.request.body)
+    index = payload['index']
+    query = payload['query']
+    cursor = payload.get('cursor')
+    limit = payload.get('limit', 20)
+    index = search.Index(name=index)
+    query_options = QueryOptions(limit=limit, cursor=cursor)
+    result = index.search(Query(query, options=query_options))
+    response = {'documents': [render_doc(document) for document in result],
+                'cursor': result.cursor}
+    self.response.headers['Content-Type'] = 'application/json'
+    self.response.out.write(json.dumps(response))
+
+
+class CleanUpHandler(webapp2.RequestHandler):
+
+  def post(self):
+    payload = json.loads(self.request.body)
+    document_ids = payload['document_ids']
+    index = payload['index']
+    idx = search.Index(name=index)
+
+    idx.delete(document_ids)
+    idx.delete_schema()
+
+urls = [
+  ('/python/search/put', PutDocumentsHandler),
+  ('/python/search/get', GetDocumentHandler),
+  ('/python/search/get-range', GetDocumentsRangeHandler),
+  ('/python/search/search', SearchDocumentsHandler),
+  ('/python/search/clean-up', CleanUpHandler),
+]

test-suite/hawkeye.py

@@ -38,7 +38,8 @@
   app_identity_tests, datastore_tests, ndb_tests, memcache_tests,
   taskqueue_tests, blobstore_tests, user_tests, images_tests, secure_url_tests,
   xmpp_tests, environment_variable_tests, async_datastore_tests, cron_tests,
-  logservice_tests, modules_tests, runtime_tests, urlfetch_tests, warmup_tests
+  logservice_tests, modules_tests, runtime_tests, urlfetch_tests, warmup_tests,
+  search_tests
 )
 
 SUPPORTED_LANGUAGES = ['java', 'python']
@@ -77,7 +78,8 @@ def build_suites_list(lang, include, exclude, application):
     'cron' : cron_tests.suite(lang, application),
     'logservice': logservice_tests.suite(lang, application),
     'modules' : modules_tests.suite(lang, application),
-    'runtime': runtime_tests.suite(lang, application)
+    'runtime': runtime_tests.suite(lang, application),
+    'search': search_tests.suite(lang, application),
   }
   # Validation include and exclude lists
   for suite_name in include + exclude:

test-suite/hawkeye_baseline_python.csv

@@ -141,3 +141,29 @@ tests.taskqueue_tests.TransactionalTaskTest.runTest,ok
 tests.urlfetch_tests.CertificateValidation.runTest,ok
 tests.user_tests.LoginURLTest.runTest,ok
 tests.xmpp_tests.SendAndReceiveTest.runTest,ok
+tests.search_tests.PutTest.test_search_put,ok
+tests.search_tests.GetTest.test_search_get,ok
+tests.search_tests.GetRangeTest.test_get_range_all_documents,ok
+tests.search_tests.GetRangeTest.test_get_range_three_documents_from_a,ok
+tests.search_tests.GetRangeTest.test_get_range_three_documents_from_document_c,ok
+tests.search_tests.SearchTest.test_search_query_AND_no_results,ok
+tests.search_tests.SearchTest.test_search_query_AND_two_results,ok
+tests.search_tests.SearchTest.test_search_query_OR,ok
+tests.search_tests.SearchTest.test_search_query_date_equal_by_field,ok
+tests.search_tests.SearchTest.test_search_query_date_global_equal,ok
+tests.search_tests.SearchTest.test_search_query_date_less_than,ok
+tests.search_tests.SearchTest.test_search_query_implicit_AND_two_results,ok
+tests.search_tests.SearchTest.test_search_query_multivalue_fields_first_value,ok
+tests.search_tests.SearchTest.test_search_query_multivalue_fields_number_no_results,ok
+tests.search_tests.SearchTest.test_search_query_multivalue_fields_second_value,ok
+tests.search_tests.SearchTest.test_search_query_multivalue_fields_string_no_results,ok
+tests.search_tests.SearchTest.test_search_query_multivalue_fields_third_value,ok
+tests.search_tests.SearchTest.test_search_query_number_by_field_equal,ok
+tests.search_tests.SearchTest.test_search_query_number_by_field_equal_colon,ok
+tests.search_tests.SearchTest.test_search_query_number_greater_than_equal_equality,ok
+tests.search_tests.SearchTest.test_search_query_number_greater_than_equal_one,ok
+tests.search_tests.SearchTest.test_search_query_number_greater_than_equal_two,ok
+tests.search_tests.SearchTest.test_search_query_number_less_than,ok
+tests.search_tests.SearchTest.test_search_query_number_less_than_equal,ok
+tests.search_tests.SearchTest.test_search_query_number_less_than_no_results,ok
+tests.search_tests.SearchTest.test_search_simple_query,ok

test-suite/hawkeye_test_runner.py

@@ -209,7 +209,7 @@ def load_report_dict_from_csv(file_name):
     A dictionary with statuses of tests (<test_id>: <status>).
   """
   with open(file_name, "r") as csv_file:
-    return {test_id: result for test_id, result in csv.reader(csv_file)}
+    return {test_id: result.rstrip() for test_id, result in csv.reader(csv_file)}
 
 
 class ReportsDiff(object):