Merge 7b90fca into cad45aa

.gitignore

@@ -4,6 +4,7 @@
 *.tab
 out
 .idea
+.svn
 *.iml
 version.json
 /results

active_data/actions/__init__.py

@@ -14,14 +14,13 @@
 from active_data import record_request
 from active_data.actions import save_query
 from jx_base import container
-from jx_elasticsearch.meta import ElasticsearchMetadata, EXPECTING_SNOWFLAKE
+from jx_elasticsearch.meta import ElasticsearchMetadata
 from jx_python.containers.list_usingPythonList import ListContainer
-from mo_dots import coalesce, is_data, set_default, split_field
 from mo_dots import is_container
-from mo_future import is_text, text_type, first
+from mo_dots import is_data, set_default, split_field
+from mo_future import is_text, first
 from mo_json import STRUCT, value2json
-from mo_logs import Log, strings
-from mo_logs.strings import expand_template, unicode2utf8
+from mo_logs import Log
 from mo_threads import Till
 from mo_times import Timer
 from mo_times.dates import Date
@@ -49,7 +48,7 @@ def send_error(active_data_timer, body, e):
     #         remove_trace(c)
     # remove_trace(e)
 
-    return Response(unicode2utf8(value2json(e)), status=status)
+    return Response(value2json(e).encode('utf8'), status=status)
 
 
 def test_mode_wait(query):

active_data/actions/json.py

@@ -17,11 +17,9 @@
 from mo_dots import listwrap, unwraplist, wrap
 from mo_json import value2json
 from mo_logs import Except, Log
-from mo_logs.strings import unicode2utf8
 from mo_math import is_integer, is_number
 from mo_threads.threads import register_thread
 from mo_times.timer import Timer
-from pyLibrary import convert
 from pyLibrary.env.flask_wrappers import cors_wrapper
 
 _keep_import = value2json
@@ -51,7 +49,7 @@ def get_raw_json(path):
 
         result.meta.active_data_response_time = active_data_timer.duration
 
-        response_data = unicode2utf8(convert.value2json(result.data, pretty=True))
+        response_data = value2json(result.data, pretty=True).encode('utf8')
         Log.note("Response is {{num}} bytes", num=len(response_data))
         return Response(
             response_data,

active_data/actions/query.py

@@ -11,6 +11,7 @@
 import flask
 from flask import Response
 
+import mo_math
 from active_data import record_request
 from active_data.actions import QUERY_TOO_LARGE, find_container, save_query, send_error, test_mode_wait
 from jx_base.container import Container
@@ -19,14 +20,12 @@
 from mo_future import binary_type
 from mo_json import json2value, value2json
 from mo_logs import Except, Log
-from mo_logs.strings import unicode2utf8, utf82unicode
-import mo_math
 from mo_threads.threads import register_thread
 from mo_times.timer import Timer
 from pyLibrary.env.flask_wrappers import cors_wrapper
 
 DEBUG = False
-BLANK = unicode2utf8(File("active_data/public/error.html").read())
+BLANK = File("active_data/public/error.html").read().encode('utf8')
 QUERY_SIZE_LIMIT = 10*1024*1024
 
 
@@ -50,16 +49,18 @@ def jx_query(path):
                     Log.error(QUERY_TOO_LARGE)
 
                 request_body = flask.request.get_data().strip()
-                text = utf82unicode(request_body)
+                text = request_body.decode('utf8')
                 data = json2value(text)
                 record_request(flask.request, data, None, None)
                 if data.meta.testing:
                     test_mode_wait(data)
 
+            find_table_timer = Timer("find container", silent=not DEBUG)
+            with find_table_timer:
+                frum = find_container(data['from'], after=None)
+
             translate_timer = Timer("translate", silent=not DEBUG)
             with translate_timer:
-                with Timer("find container", silent=not DEBUG):
-                    frum = find_container(data['from'], after=None)
                 result = jx.run(data, container=frum)
 
                 if isinstance(result, Container):  # TODO: REMOVE THIS CHECK, jx SHOULD ALWAYS RETURN Containers
@@ -73,13 +74,14 @@ def jx_query(path):
                     except Exception as e:
                         Log.warning("Unexpected save problem", cause=e)
 
+            result.meta.timing.find_table = mo_math.round(find_table_timer.duration.seconds, digits=4)
             result.meta.timing.preamble = mo_math.round(preamble_timer.duration.seconds, digits=4)
             result.meta.timing.translate = mo_math.round(translate_timer.duration.seconds, digits=4)
             result.meta.timing.save = mo_math.round(save_timer.duration.seconds, digits=4)
             result.meta.timing.total = "{{TOTAL_TIME}}"  # TIMING PLACEHOLDER
 
             with Timer("jsonification", silent=not DEBUG) as json_timer:
-                response_data = unicode2utf8(value2json(result))
+                response_data = value2json(result).encode('utf8')
 
         with Timer("post timer", silent=not DEBUG):
             # IMPORTANT: WE WANT TO TIME OF THE JSON SERIALIZATION, AND HAVE IT IN THE JSON ITSELF.

active_data/actions/save_query.py

@@ -20,7 +20,6 @@
 from mo_kwargs import override
 from mo_logs import Log
 from mo_logs.exceptions import Except
-from mo_logs.strings import unicode2utf8
 from mo_threads import Thread
 from mo_threads.threads import register_thread
 from mo_times.dates import Date
@@ -53,14 +52,14 @@ def find_query(hash):
             )
         else:
             return Response(
-                unicode2utf8(query),
+                query.encode('utf8'),
                 status=200
             )
     except Exception as e:
         e = Except.wrap(e)
         Log.warning("problem finding query with hash={{hash}}", hash=hash, cause=e)
         return Response(
-            unicode2utf8(convert.value2json(e)),
+            value2json(e).encode('utf8'),
             status=400
         )
 
@@ -113,7 +112,7 @@ def save(self, query):
         """
         query.meta = None
         json = convert.value2json(query)
-        hash = unicode2utf8(json)
+        hash = json.encode('utf8')
 
         # TRY MANY HASHES AT ONCE
         hashes = [None] * HASH_BLOCK_SIZE

active_data/actions/sql.py

@@ -11,21 +11,20 @@
 import flask
 from flask import Response
 
+import mo_math
+import moz_sql_parser
 from active_data import record_request
 from active_data.actions import find_container, save_query, send_error, test_mode_wait
 from active_data.actions.query import BLANK, QUERY_SIZE_LIMIT
 from jx_base.container import Container
 from jx_python import jx
 from mo_dots import is_data, is_list, listwrap, unwraplist, wrap
-from mo_json import json2value, utf82unicode, value2json
+from mo_json import json2value, value2json
 from mo_logs import Log
 from mo_logs.exceptions import Except
-from mo_logs.strings import unicode2utf8
-import mo_math
 from mo_testing.fuzzytestcase import assertAlmostEqual
 from mo_threads.threads import register_thread
 from mo_times.timer import Timer
-import moz_sql_parser
 from pyLibrary.env.flask_wrappers import cors_wrapper
 
 
@@ -51,7 +50,7 @@ def sql_query(path):
                     Log.error("Query is too large")
 
                 request_body = flask.request.get_data().strip()
-                text = utf82unicode(request_body)
+                text = request_body.decode('utf8')
                 data = json2value(text)
                 record_request(flask.request, data, None, None)
 
@@ -85,7 +84,7 @@ def sql_query(path):
             result.meta.timing.total = "{{TOTAL_TIME}}"  # TIMING PLACEHOLDER
 
             with Timer("jsonification", silent=True) as json_timer:
-                response_data = unicode2utf8(value2json(result))
+                response_data = value2json(result).encode('utf8')
 
         with Timer("post timer", silent=True):
             # IMPORTANT: WE WANT TO TIME OF THE JSON SERIALIZATION, AND HAVE IT IN THE JSON ITSELF.

active_data/app.py

@@ -8,11 +8,11 @@
 #
 from __future__ import absolute_import, division, unicode_literals
 
-from _ssl import PROTOCOL_SSLv23
 import os
 from ssl import SSLContext
 
 import flask
+from _ssl import PROTOCOL_SSLv23
 from flask import Flask, Response
 
 import active_data
@@ -27,13 +27,12 @@
 from jx_base import container
 from mo_dots import is_data
 from mo_files import File, TempFile
-from mo_future import text_type
+from mo_future import text
 from mo_logs import Log, constants, machine_metadata, startup
-from mo_logs.strings import unicode2utf8
 from mo_threads import Thread, stop_main_thread
 from mo_threads.threads import MAIN_THREAD, register_thread
 from pyLibrary.env import elasticsearch, http
-from pyLibrary.env.flask_wrappers import cors_wrapper, dockerflow
+from pyLibrary.env.flask_wrappers import cors_wrapper, dockerflow, add_version
 
 
 class ActiveDataApp(Flask):
@@ -88,7 +87,7 @@ def _default(path):
     record_request(flask.request, None, flask.request.get_data(), None)
 
     return Response(
-        unicode2utf8(OVERVIEW),
+        OVERVIEW.encode('utf8'),
         status=200,
         headers={
             "Content-Type": "text/html"
@@ -116,7 +115,7 @@ def setup():
     constants.set(config.constants)
     Log.start(config.debug)
 
-    File.new_instance("activedata.pid").write(text_type(machine_metadata.pid))
+    File.new_instance("activedata.pid").write(text(machine_metadata.pid))
 
     # PIPE REQUEST LOGS TO ES DEBUG
     if config.request_logs:
@@ -126,8 +125,11 @@ def setup():
 
     if config.dockerflow:
         def backend_check():
-            http.get_json(config.elasticsearch.host + ":" + text_type(config.elasticsearch.port))
+            http.get_json(config.elasticsearch.host + ":" + text(config.elasticsearch.port))
         dockerflow(flask_app, backend_check)
+    else:
+        # IF NOT USING DOCKERFLOW, THEN RESPOND WITH A SIMPLER __version__
+        add_version(flask_app)
 
     # SETUP DEFAULT CONTAINER, SO THERE IS SOMETHING TO QUERY
     container.config.default = {
@@ -202,7 +204,7 @@ def _exit():
     Log.note("Got request to shutdown")
     try:
         return Response(
-            unicode2utf8(OVERVIEW),
+            OVERVIEW.encode('utf8'),
             status=400,
             headers={
                 "Content-Type": "text/html"