From 9a22e2e624eb4c605a18882691b71e28974649cc Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Fri, 27 Nov 2015 13:00:36 +0900 Subject: [PATCH 1/3] TAJO-1753: GlobalEngine causes NPE occurs occasionally. --- .../src/main/java/org/apache/tajo/master/TajoMaster.java | 4 ++-- .../src/main/java/org/apache/tajo/worker/TajoWorker.java | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index a8a84feb95..b7646415d4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -203,6 +203,8 @@ public void serviceInit(Configuration conf) throws Exception { addIfService(restServer); PythonScriptEngine.initPythonScriptEngineFiles(); + + initSystemMetrics(); // Try to start up all services in TajoMaster. // If anyone is failed, the master prints out the errors and immediately should shutdowns @@ -334,8 +336,6 @@ public void serviceStart() throws Exception { } initWebServer(); - initSystemMetrics(); - haService = ServiceTrackerFactory.get(systemConf); haService.register(); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java index c7cac4f2e3..c50621899d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java @@ -33,10 +33,8 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.CatalogClient; import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.function.FunctionLoader; -import org.apache.tajo.function.FunctionSignature; import org.apache.tajo.master.cluster.WorkerConnectionInfo; import org.apache.tajo.metrics.Node; import org.apache.tajo.plan.function.python.PythonScriptEngine; @@ -199,6 +197,7 @@ public void serviceInit(Configuration conf) throws Exception { httpPort = initWebServer(); } + initWorkerMetrics(); super.serviceInit(conf); int pullServerPort; @@ -327,7 +326,6 @@ public void serviceStart() throws Exception { initCleanupService(); } - initWorkerMetrics(); super.serviceStart(); LOG.info("Tajo Worker is started"); } From 3a0e7b2ef6ed9d417bbee17f10521a2289feca22 Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Fri, 27 Nov 2015 14:21:10 +0900 Subject: [PATCH 2/3] fix NPE --- .../src/main/java/org/apache/tajo/master/TajoMaster.java | 4 ++-- .../src/main/java/org/apache/tajo/worker/TajoWorker.java | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index b7646415d4..fb20a03c98 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -204,8 +204,6 @@ public void serviceInit(Configuration conf) throws Exception { PythonScriptEngine.initPythonScriptEngineFiles(); - initSystemMetrics(); - // Try to start up all services in TajoMaster. // If anyone is failed, the master prints out the errors and immediately should shutdowns try { @@ -323,6 +321,7 @@ public void serviceStart() throws Exception { // check base tablespace and databases checkBaseTBSpaceAndDatabase(); + initSystemMetrics(); super.serviceStart(); // Setting the system global configs @@ -336,6 +335,7 @@ public void serviceStart() throws Exception { } initWebServer(); + haService = ServiceTrackerFactory.get(systemConf); haService.register(); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java index c50621899d..c7cac4f2e3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java @@ -33,8 +33,10 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.CatalogClient; import org.apache.tajo.catalog.CatalogService; +import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.function.FunctionLoader; +import org.apache.tajo.function.FunctionSignature; import org.apache.tajo.master.cluster.WorkerConnectionInfo; import org.apache.tajo.metrics.Node; import org.apache.tajo.plan.function.python.PythonScriptEngine; @@ -197,7 +199,6 @@ public void serviceInit(Configuration conf) throws Exception { httpPort = initWebServer(); } - initWorkerMetrics(); super.serviceInit(conf); int pullServerPort; @@ -326,6 +327,7 @@ public void serviceStart() throws Exception { initCleanupService(); } + initWorkerMetrics(); super.serviceStart(); LOG.info("Tajo Worker is started"); } From 6c1c0efed25737ee817cf3b6ab093cc7dc9de13c Mon Sep 17 00:00:00 2001 From: Jinho Kim Date: Fri, 27 Nov 2015 17:54:36 +0900 Subject: [PATCH 3/3] fix NPE --- .../main/java/org/apache/tajo/master/GlobalEngine.java | 9 +++++---- .../src/main/java/org/apache/tajo/master/TajoMaster.java | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 1966244535..7f18fd4fbb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -36,13 +36,13 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.*; import org.apache.tajo.master.TajoMaster.MasterContext; import org.apache.tajo.master.exec.DDLExecutor; import org.apache.tajo.master.exec.QueryExecutor; import org.apache.tajo.metrics.Master; +import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.plan.LogicalOptimizer; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.LogicalPlanner; @@ -54,6 +54,7 @@ import org.apache.tajo.session.Session; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.CommonTestingUtil; +import org.apache.tajo.util.metrics.TajoSystemMetrics; import java.sql.SQLException; import java.util.concurrent.ExecutionException; @@ -176,9 +177,9 @@ public SubmitQueryResponse executeQuery(Session session, String query, boolean i LOG.info("Query: " + query); QueryContext queryContext = createQueryContext(session); Expr planningContext; - + TajoSystemMetrics metrics = context.getMetrics(); try { - context.getMetrics().counter(Master.Query.SUBMITTED).inc(); + if(metrics != null) metrics.counter(Master.Query.SUBMITTED).inc(); if (isJson) { planningContext = buildExpressionFromJson(query); @@ -195,7 +196,7 @@ public SubmitQueryResponse executeQuery(Session session, String query, boolean i } catch (Throwable t) { ExceptionUtil.printStackTraceIfError(LOG, t); - context.getMetrics().counter(Master.Query.ERROR).inc(); + if(metrics != null) metrics.counter(Master.Query.ERROR).inc(); SubmitQueryResponse.Builder responseBuilder = SubmitQueryResponse.newBuilder(); responseBuilder.setUserName(queryContext.get(SessionVars.USERNAME)); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index fb20a03c98..a74573ef62 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -321,8 +321,8 @@ public void serviceStart() throws Exception { // check base tablespace and databases checkBaseTBSpaceAndDatabase(); - initSystemMetrics(); super.serviceStart(); + initSystemMetrics(); // Setting the system global configs systemConf.setSocketAddr(ConfVars.CATALOG_ADDRESS.varname,