From 09387b86df03cb1a53fd82da650038a8a0dd364a Mon Sep 17 00:00:00 2001 From: Byunghwa Yun Date: Wed, 6 Jan 2016 15:24:26 +0900 Subject: [PATCH 1/5] TAJO-2036: Prevent out of memory in the master server, if the query result is large --- .../org/apache/tajo/webapp/QueryExecutorServlet.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java index 896c83e20c..b857e50747 100644 --- a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java +++ b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java @@ -17,6 +17,7 @@ import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.jdbc.FetchResultSet; import org.apache.tajo.service.ServiceTrackerFactory; +import org.apache.tajo.util.Bytes; import org.apache.tajo.util.JSPUtil; import org.apache.tajo.util.TajoIdUtils; import org.codehaus.jackson.map.DeserializationConfig; @@ -507,14 +508,20 @@ private void MakeResultText(ResultSet res, TableDesc desc) throws SQLException { numOfRows = resultRows; } + int currentResultSize = 0; int rowCount = 0; while (res.next()) { - if(rowCount > numOfRows) { + if(rowCount > numOfRows || currentResultSize > sizeLimit) { break; } List row = new ArrayList<>(); for(int i = 0; i < numOfColumns; i++) { - row.add(String.valueOf(res.getObject(i + 1))); + String columnValue = String.valueOf(res.getObject(i + 1)); + try { + currentResultSize += columnValue.getBytes(Bytes.UTF8_ENCODING).length; + } catch (Exception e) { + } + row.add(columnValue); } queryResult.add(row); rowCount++; From d70b07f9443ce58292dc421ad66112023557f1ba Mon Sep 17 00:00:00 2001 From: Byunghwa Yun Date: Wed, 6 Jan 2016 18:46:52 +0900 Subject: [PATCH 2/5] numOfRows is not used. --- .../org/apache/tajo/webapp/QueryExecutorServlet.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java index b857e50747..0d1fab9b1a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java +++ b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java @@ -198,7 +198,6 @@ public void service(HttpServletRequest request, errorResponse(response, queryRunner.error); return; } - returnValue.put("numOfRows", queryRunner.numOfRows); returnValue.put("resultSize", queryRunner.resultRows); returnValue.put("resultData", queryRunner.queryResult); returnValue.put("resultColumns", queryRunner.columnNames); @@ -287,7 +286,6 @@ class QueryRunner extends Thread { String database; long resultRows; int sizeLimit; - long numOfRows; Exception error; AtomicInteger progress = new AtomicInteger(0); @@ -502,16 +500,10 @@ private void MakeResultText(ResultSet res, TableDesc desc) throws SQLException { } queryResult = new ArrayList<>(); - if(sizeLimit < resultRows) { - numOfRows = (long)((float)(resultRows) * ((float)sizeLimit / (float) resultRows)); - } else { - numOfRows = resultRows; - } - int currentResultSize = 0; int rowCount = 0; while (res.next()) { - if(rowCount > numOfRows || currentResultSize > sizeLimit) { + if(currentResultSize > sizeLimit) { break; } List row = new ArrayList<>(); From 47fcae5d214e54d4dadd0ad1871a119b0efdcf53 Mon Sep 17 00:00:00 2001 From: Byunghwa Yun Date: Fri, 22 Jan 2016 18:02:53 +0900 Subject: [PATCH 3/5] add row limit field. --- .../org/apache/tajo/webapp/QueryExecutorServlet.java | 8 +++++++- .../main/resources/webapps/admin/query_executor.jsp | 10 +++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java index 0d1fab9b1a..ae954de1cb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java +++ b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java @@ -162,6 +162,11 @@ public void service(HttpServletRequest request, } catch (java.lang.NumberFormatException nfe) { queryRunner.sizeLimit = 1048576; } + try { + queryRunner.rowLimit = Integer.parseInt(request.getParameter("limitRow")); + } catch (java.lang.NumberFormatException nfe) { + queryRunner.rowLimit = 3000000; + } synchronized(queryRunners) { queryRunners.put(queryRunnerId, queryRunner); } @@ -286,6 +291,7 @@ class QueryRunner extends Thread { String database; long resultRows; int sizeLimit; + long rowLimit; Exception error; AtomicInteger progress = new AtomicInteger(0); @@ -503,7 +509,7 @@ private void MakeResultText(ResultSet res, TableDesc desc) throws SQLException { int currentResultSize = 0; int rowCount = 0; while (res.next()) { - if(currentResultSize > sizeLimit) { + if(rowCount > rowLimit || currentResultSize > sizeLimit) { break; } List row = new ArrayList<>(); diff --git a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp index 6b6a3dba9f..280f01b550 100644 --- a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp +++ b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp @@ -64,6 +64,7 @@ var progressTimer = null; var queryRunnerId = null; var PRINT_LIMIT = 25; var SIZE_LIMIT = 104857600; // Limit size of displayed results.(Bytes) +var ROW_LIMIT = 3000000; var pageNum = 0; var pageCount, storedColumns, storedData; @@ -87,6 +88,11 @@ function runQuery() { } else if(Math.ceil(Number($("#sizeLimit").val())) > 0) { SIZE_LIMIT = Number($("#sizeLimit").val()) * 1024 * 1024; } + if(Math.ceil(Number($("#rowLimit").val())) >= 10) { + ROW_LIMIT = 10 * 1000 * 1000 - 1; + } else if(Math.ceil(Number($("#rowLimit").val())) > 0) { + ROW_LIMIT = Number($("#rowLimit").val()) * 1000 * 1000; + } if(Math.ceil(Number($("#printLimit").val())) > 0) { PRINT_LIMIT = Number($("#printLimit").val()); } @@ -101,7 +107,7 @@ function runQuery() { $.ajax({ type: "POST", url: "query_exec", - data: { action: "runQuery", query: query, prevQueryId: queryRunnerId, limitSize:SIZE_LIMIT, database: sbox.options[sbox.selectedIndex].text } + data: { action: "runQuery", query: query, prevQueryId: queryRunnerId, limitSize:SIZE_LIMIT, limitRow:ROW_LIMIT, database: sbox.options[sbox.selectedIndex].text } }) .done(function(msg) { var resultJson = $.parseJSON(msg); @@ -310,6 +316,8 @@ function getPage() {

Limit : MB

+ Limit Rows : M +

Rows/Page :

From 196a6499e63e55a2590babdf62fb16ae80afd055 Mon Sep 17 00:00:00 2001 From: Byunghwa Yun Date: Fri, 22 Jan 2016 18:06:57 +0900 Subject: [PATCH 4/5] Remove tabs. --- .../tajo/webapp/QueryExecutorServlet.java | 6 +-- .../webapps/admin/query_executor.jsp | 46 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java index ae954de1cb..708c552207 100644 --- a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java +++ b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java @@ -516,9 +516,9 @@ private void MakeResultText(ResultSet res, TableDesc desc) throws SQLException { for(int i = 0; i < numOfColumns; i++) { String columnValue = String.valueOf(res.getObject(i + 1)); try { - currentResultSize += columnValue.getBytes(Bytes.UTF8_ENCODING).length; - } catch (Exception e) { - } + currentResultSize += columnValue.getBytes(Bytes.UTF8_ENCODING).length; + } catch (Exception e) { + } row.add(columnValue); } queryResult.add(row); diff --git a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp index 280f01b550..1d5d1a33ce 100644 --- a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp +++ b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp @@ -240,26 +240,26 @@ function getCSV() { } function getNext() { - var printedLine = 0; - if(pageCount > pageNum) { - pageNum++; - document.getElementById("selectPage").options.selectedIndex = pageNum; - }else { - alert("There's no next page."); - return; - } - getPage(); + var printedLine = 0; + if(pageCount > pageNum) { + pageNum++; + document.getElementById("selectPage").options.selectedIndex = pageNum; + } else { + alert("There's no next page."); + return; + } + getPage(); } function getPrev() { - if(pageNum > 0 ) { - pageNum--; - document.getElementById("selectPage").options.selectedIndex = pageNum; - } else { - alert("There's no previous page."); - return; - } - getPage(); + if(pageNum > 0 ) { + pageNum--; + document.getElementById("selectPage").options.selectedIndex = pageNum; + } else { + alert("There's no previous page."); + return; + } + getPage(); } function getSelectedPage() { @@ -304,12 +304,12 @@ function getPage() { Database :

From 5d89d06200d59590a764f58b8a3258cb3dcfbc2f Mon Sep 17 00:00:00 2001 From: Byunghwa Yun Date: Thu, 28 Jan 2016 14:04:53 +0900 Subject: [PATCH 5/5] Remove max row limit. --- tajo-core/src/main/resources/webapps/admin/query_executor.jsp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp index 1d5d1a33ce..f359c994c0 100644 --- a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp +++ b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp @@ -88,9 +88,7 @@ function runQuery() { } else if(Math.ceil(Number($("#sizeLimit").val())) > 0) { SIZE_LIMIT = Number($("#sizeLimit").val()) * 1024 * 1024; } - if(Math.ceil(Number($("#rowLimit").val())) >= 10) { - ROW_LIMIT = 10 * 1000 * 1000 - 1; - } else if(Math.ceil(Number($("#rowLimit").val())) > 0) { + if(Math.ceil(Number($("#rowLimit").val())) > 0) { ROW_LIMIT = Number($("#rowLimit").val()) * 1000 * 1000; } if(Math.ceil(Number($("#printLimit").val())) > 0) {