Skip to content
Permalink
Browse files

HUE-2387 [beeswax] Support getting logs using FetchResults() api

  • Loading branch information...
pslawski authored and romainr committed Oct 11, 2014
1 parent 37c80cb commit 6a0246710f7deeb0fd2e1f2b3b209ad119c30b72

Some generated files are not rendered by default. Learn more.

@@ -172,7 +172,8 @@ def watch_query_refresh_json(request, id):
handle, state = _get_query_handle_and_state(query_history)

try:
log = db.get_log(handle)
start_over = request.POST.get('log-start-over') == 'true'
log = db.get_log(handle, start_over=start_over)
except Exception, ex:
log = str(ex)

@@ -60,6 +60,14 @@
type=int,
help=_t('Timeout in seconds for Thrift calls.'))

USE_GET_LOG_API = Config(
key='use_get_log_api',
default=True,
type=coerce_bool,
help=_t('Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs.'
'If false, Hue will use the FetchResults() thrift call instead.')
)

BROWSE_PARTITIONED_TABLE_LIMIT = Config(
key='browse_partitioned_table_limit',
default=250,
@@ -338,8 +338,8 @@ def use(self, database):
return self.client.use(query)


def get_log(self, query_handle):
return self.client.get_log(query_handle)
def get_log(self, query_handle, start_over=True):
return self.client.get_log(query_handle, start_over)


def get_state(self, handle):
@@ -14,6 +14,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from itertools import imap

import logging
import re
@@ -590,6 +591,15 @@ def fetch_result(self, operation_handle, orientation=TFetchOrientation.FETCH_NEX
return res, schema


def fetch_log(self, operation_handle, orientation=TFetchOrientation.FETCH_NEXT, max_rows=1000):
req = TFetchResultsReq(operationHandle=operation_handle, orientation=orientation,
maxRows=max_rows, fetchType=1)
res = self.call(self._client.FetchResults, req)

lines = imap(lambda r: r.colVals[0].stringVal.value, res.results.rows)
return '\n'.join(lines)


def get_operation_status(self, operation_handle):
req = TGetOperationStatusReq(operationHandle=operation_handle)
return self.call(self._client.GetOperationStatus, req)
@@ -759,9 +769,18 @@ def dump_config(self):
return 'Does not exist in HS2'


def get_log(self, handle):
def get_log(self, handle, start_over=True):
operationHandle = handle.get_rpc_handle()
return self._client.get_log(operationHandle)

if beeswax_conf.USE_GET_LOG_API.get() or self.query_server['server_name'] == 'impala':
return self._client.get_log(operationHandle)
else:
if start_over:
orientation = TFetchOrientation.FETCH_FIRST
else:
orientation = TFetchOrientation.FETCH_NEXT

return self._client.fetch_log(operationHandle, orientation=orientation, max_rows=-1)


def get_databases(self):
@@ -2637,6 +2637,11 @@ function setupCodeMirrorSubscription() {
// Knockout
viewModel = new BeeswaxViewModel("${app_name}");
% if not beeswax_conf.USE_GET_LOG_API.get() and app_name != 'impala':
viewModel.shouldAppendLogs = true;
% endif
% if query_history:
loadQueryHistory(${query_history.id});
% elif design.id:
@@ -560,7 +560,8 @@ function BeeswaxViewModel(server) {
self.watchQuery = function() {
var data = {
'query-query': self.design.query.value(),
'query-database': self.database()
'query-database': self.database(),
'log-start-over': self.design.watch.logs().length == 0
};
$.extend(data, self.getSettingsFormData());
$.extend(data, self.getFileResourcesFormData());
@@ -589,6 +590,17 @@ function BeeswaxViewModel(server) {
$.ajax(request);
};

self.shouldAppendLogs = false;

self.applyLogs = function(log) {
var lines = log.split("\n")

if (self.shouldAppendLogs) {
lines = self.design.watch.logs().concat(lines);
}
self.design.watch.logs(lines);
};

self.watchQueryLoop = function(fn) {
var TIMEOUT = 100;
var timer = null;
@@ -607,7 +619,7 @@ function BeeswaxViewModel(server) {
clearTimeout(timer);
self.design.isRunning(false);
if (data.log) {
self.design.watch.logs(data.log.split("\n"));
self.applyLogs(data.log)
// scroll logs
self.design.watch.jobUrls(data.jobUrls);
}
@@ -623,7 +635,7 @@ function BeeswaxViewModel(server) {
} else {
self.design.statement(data.statement); // In case new no result statement executed
if (data.log) {
self.design.watch.logs(data.log.split("\n"));
self.applyLogs(data.log)
// scroll logs
self.design.watch.jobUrls(data.jobUrls);
}
@@ -1002,6 +1002,9 @@ struct TFetchResultsReq {
// Max number of rows that should be returned in
// the rowset.
3: required i64 maxRows

// The type of a fetch results request. 0 represents Query output. 1 represents Log
4: optional i16 fetchType = 0
}

struct TFetchResultsResp {
@@ -724,6 +724,10 @@
# Timeout in seconds for thrift calls to Hive service
## server_conn_timeout=120

# Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs.
# If false, Hue will use the FetchResults() thrift call instead.
## use_get_log_api=true

# Set a LIMIT clause when browsing a partitioned table.
# A positive value will be set as the LIMIT. If 0 or negative, do not set any limit.
## browse_partitioned_table_limit=250
@@ -731,6 +731,10 @@
# Timeout in seconds for thrift calls to Hive service
## server_conn_timeout=120

# Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs.
# If false, Hue will use the FetchResults() thrift call instead.
## use_get_log_api=true

# Set a LIMIT clause when browsing a partitioned table.
# A positive value will be set as the LIMIT. If 0 or negative, do not set any limit.
## browse_partitioned_table_limit=250

0 comments on commit 6a02467

Please sign in to comment.
You can’t perform that action at this time.