diff --git a/oss_src/sframe/CMakeLists.txt b/oss_src/sframe/CMakeLists.txt index f1fe9725..9d85f719 100644 --- a/oss_src/sframe/CMakeLists.txt +++ b/oss_src/sframe/CMakeLists.txt @@ -55,5 +55,7 @@ make_copy_target(_local_sys_util_ pylambda_worker_lib ) add_dependencies(spark_unity _local_sys_util_) + file(DOWNLOAD http://s3-us-west-2.amazonaws.com/glbin-engine/spark_unity_0.4.jar ${CMAKE_CURRENT_BINARY_DIR}/spark_unity.jar EXPECTED_MD5 1ff73889a5484a90b0f1675641c16488) + diff --git a/oss_src/unity/python/sframe/data_structures/sketch.py b/oss_src/unity/python/sframe/data_structures/sketch.py index 1ce78750..d76b95ee 100644 --- a/oss_src/unity/python/sframe/data_structures/sketch.py +++ b/oss_src/unity/python/sframe/data_structures/sketch.py @@ -460,7 +460,6 @@ def quantile(self, quantile_val): out : float | str An estimate of the value at a quantile. """ - _mt._get_metric_tracker().track('sketch.quantile.%g' % quantile_val) with cython_context(): return self.__proxy__.get_quantile(quantile_val) diff --git a/oss_src/unity/python/sframe/toolkits/_main.py b/oss_src/unity/python/sframe/toolkits/_main.py index bc17f59b..02e3284d 100644 --- a/oss_src/unity/python/sframe/toolkits/_main.py +++ b/oss_src/unity/python/sframe/toolkits/_main.py @@ -55,8 +55,6 @@ def run(toolkit_name, options, verbose=True, show_progress=False): if (not verbose): glconnect.get_client().set_log_progress(False) # spawn progress threads - server_addr = glconnect.get_server().get_server_addr() - splits = server_addr.split(':') try: start_time = time.time() (success, message, params) = unity.run_toolkit(toolkit_name, options) @@ -85,4 +83,7 @@ def run(toolkit_name, options, verbose=True, show_progress=False): if success: return params else: + metric_name = 'toolkit.%s.toolkit_error' % (toolkit_name) + _get_metric_tracker().track(metric_name, value=1, properties=track_props, send_sys_info=False) + raise ToolkitError(str(message)) diff --git a/oss_src/unity/python/sframe/util/__init__.py b/oss_src/unity/python/sframe/util/__init__.py index 5e7d2688..6abbc2b2 100644 --- a/oss_src/unity/python/sframe/util/__init__.py +++ b/oss_src/unity/python/sframe/util/__init__.py @@ -27,6 +27,12 @@ import datetime as _datetime import time as _time import logging as _logging +import sys as _sys + +def _i_am_a_lambda_worker(): + if _re.match(".*lambda_worker.*", _sys.argv[0]) is not None: + return True + return False __LOGGER__ = _logging.getLogger(__name__) @@ -70,8 +76,12 @@ # Set module specific log levels logging.getLogger('librato').setLevel(logging.CRITICAL) logging.getLogger('requests').setLevel(logging.CRITICAL) -logging.getLogger(root_package_name).setLevel(logging.INFO) -logging.getLogger(__name__).setLevel(logging.INFO) +if _i_am_a_lambda_worker(): + logging.getLogger(root_package_name).setLevel(logging.WARNING) + logging.getLogger(__name__).setLevel(logging.WARNING) +else: + logging.getLogger(root_package_name).setLevel(logging.INFO) + logging.getLogger(__name__).setLevel(logging.INFO) #amend the logging configuration with a handler streaming to a message queue