From 888505d462d79e86c34d3ea8383e4207153f2731 Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Fri, 4 Sep 2015 17:47:17 +0900 Subject: [PATCH 01/90] TAJO-1740: Update Partition Table document --- .../partitioning/column_partitioning.rst | 187 ++++++++++++++++-- 1 file changed, 175 insertions(+), 12 deletions(-) diff --git a/tajo-docs/src/main/sphinx/partitioning/column_partitioning.rst b/tajo-docs/src/main/sphinx/partitioning/column_partitioning.rst index 4b8d6bf9b4..ecb658c56e 100644 --- a/tajo-docs/src/main/sphinx/partitioning/column_partitioning.rst +++ b/tajo-docs/src/main/sphinx/partitioning/column_partitioning.rst @@ -11,22 +11,40 @@ How to Create a Column Partitioned Table You can create a partitioned table by using the ``PARTITION BY`` clause. For a column partitioned table, you should use the ``PARTITION BY COLUMN`` clause with partition keys. -For example, assume there is a table ``orders`` composed of the following schema. :: +For example, assume there is a table ``student`` composed of the following schema. - id INT, - item_name TEXT, - price FLOAT +.. code-block:: sql + + id INT, + name TEXT, + grade TEXT -Also, assume that you want to use ``order_date TEXT`` and ``ship_date TEXT`` as the partition keys. -Then, you should create a table as follows: +Now you want to partition on country. Your Tajo definition would be this: .. code-block:: sql - CREATE TABLE orders ( - id INT, - item_name TEXT, - price - ) PARTITION BY COLUMN (order_date TEXT, ship_date TEXT); + CREATE TABLE student ( + id INT, + name TEXT, + grade TEXT + ) PARTITION BY COLUMN (country TEXT); + +Now your users will still query on ``WHERE country = '...'`` but the 2nd column will be the original values. +Here's an example statement to create a table: + +.. code-block:: sql + + CREATE TABLE student ( + id INT, + name TEXT, + grade TEXT + ) USING PARQUET + PARTITION BY COLUMN (country TEXT, city TEXT); + +The statement above creates the student table with id, name, grade. The table is also partitioned and data is stored in parquet files. + +You might have noticed that while the partitioning key columns are a part of the table DDL, they’re only listed in the ``PARTITION BY`` clause. In Tajo, as data is written to disk, each partition of data will be automatically split out into different folders, e.g. country=USA/city=NEWYORK. During a read operation, Tajo will use the folder structure to quickly locate the right partitions and also return the partitioning columns as columns in the result set. + ================================================== Partition Pruning on Column Partitioned Tables @@ -44,9 +62,154 @@ during query planning phase. * LIKE predicates with a leading wild-card character * IN list predicates +.. code-block:: sql + + SELECT * FROM student WHERE country = 'KOREA' AND city = 'SEOUL'; + SELECT * FROM student WHERE country = 'USA' AND (city = 'NEWYORK' OR city = 'BOSTON'); + SELECT * FROM student WHERE country = 'USA' AND city <> 'NEWYORK'; + + +================================================== +Add data to Partition Table +================================================== + +Tajo provides a very useful feature of dynamic partitioning. You don't need to use any syntax with both ``INSERT INTO ... SELECT`` and ``Create Table As Select(CTAS)`` statments for dynamic partitioning. Tajo will automatically filter the data, create directories, move filtered data to appropriate directory and create partition over it. + +For example, assume there are both ``student_source`` and ``student`` tables composed of the following schema. + +.. code-block:: sql + + CREATE TABLE student_source ( + id INT, + name TEXT, + gender char(1), + grade TEXT, + country TEXT, + city TEXT, + phone TEXT + ); + + CREATE TABLE student ( + id INT, + name TEXT, + grade TEXT + ) PARTITION BY COLUMN (country TEXT, city TEXT); + + +How to INSERT dynamically to partition table +-------------------------------------------------------- + +If you want to load an entire country or an entire city in one fell swoop: + +.. code-block:: sql + + INSERT OVERWRITE INTO student + SELECT id, name, grade, country, city + FROM student_source; + + +How to CTAS dynamically to partition table +-------------------------------------------------------- + +when a partition table is created: + +.. code-block:: sql + + DROP TABLE if exists student; + + CREATE TABLE student ( + id INT, + name TEXT, + grade TEXT + ) PARTITION BY COLUMN (country TEXT, city TEXT) + AS SELECT id, name, grade, country, city + FROM student_source; + + +.. note:: + + When loading data into a partition, it’s necessary to include the partition columns as the last columns in the query. The column names in the source query don’t need to match the partition column names. + + ================================================== Compatibility Issues with Apache Hive™ ================================================== If partitioned tables of Hive are created as external tables in Tajo, Tajo can process the Hive partitioned tables directly. -There haven't known compatibility issues yet. \ No newline at end of file + + +How to create partition table +-------------------------------------------------------- + +If you create a partition table as follows in Tajo: + +.. code-block:: sql + + default> CREATE TABLE student ( + id INT, + name TEXT, + grade TEXT + ) PARTITION BY COLUMN (country TEXT, city TEXT); + + +And then you can get table information in Hive: + +.. code-block:: sql + + hive> desc student; + OK + id int + name string + grade string + country string + city string + + # Partition Information + # col_name data_type comment + + country string + city string + + +Or as you create the table in Hive: + +.. code-block:: sql + + hive > CREATE TABLE student ( + id int, + name string, + grade string + ) PARTITIONED BY (country string, city string) + ROW FORMAT DELIMITED + FIELDS TERMINATED BY '|' ; + +You will see table information in Tajo: + +.. code-block:: sql + + default> \d student; + table name: default.student + table uri: hdfs://your_hdfs_namespace/user/hive/warehouse/student + store type: TEXT + number of rows: 0 + volume: 0 B + Options: + 'text.null'='\\N' + 'transient_lastDdlTime'='1438756422' + 'text.delimiter'='|' + + schema: + id INT4 + name TEXT + grade TEXT + + Partitions: + type:COLUMN + columns::default.student.country (TEXT), default.student.city (TEXT) + + +How to add data to partition table +-------------------------------------------------------- + +In Tajo, you can add data dynamically to partition table of Hive with both ``INSERT INTO ... SELECT`` and ``Create Table As Select (CTAS)`` statments. Tajo will automatically filter the data to HiveMetastore, create directories and move filtered data to appropriate directory on the distributed file system + From 4c46d205b05311203d6f1ac4a1d23a8ffbfd9aaf Mon Sep 17 00:00:00 2001 From: JaeHwa Jung Date: Wed, 23 Sep 2015 10:31:21 +0900 Subject: [PATCH 02/90] Update trivial comment --- tajo-docs/src/main/sphinx/partitioning/column_partitioning.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tajo-docs/src/main/sphinx/partitioning/column_partitioning.rst b/tajo-docs/src/main/sphinx/partitioning/column_partitioning.rst index ecb658c56e..af6dc0ce45 100644 --- a/tajo-docs/src/main/sphinx/partitioning/column_partitioning.rst +++ b/tajo-docs/src/main/sphinx/partitioning/column_partitioning.rst @@ -11,7 +11,7 @@ How to Create a Column Partitioned Table You can create a partitioned table by using the ``PARTITION BY`` clause. For a column partitioned table, you should use the ``PARTITION BY COLUMN`` clause with partition keys. -For example, assume there is a table ``student`` composed of the following schema. +For example, assume a table with the following schema. .. code-block:: sql From 1eb10045908a8c5c6304e10379fa08a68bd61dad Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 22 Sep 2015 19:20:18 -0700 Subject: [PATCH 03/90] TAJO-1860: Refactor Rpc clients to take Connection Parameters. Closes #763 --- CHANGES | 2 + .../apache/tajo/catalog/CatalogClient.java | 14 +- .../tajo/cli/tsql/CliClientParamsFactory.java | 50 ++++++ .../org/apache/tajo/cli/tsql/TajoCli.java | 22 ++- .../tajo/client/ClientParameterHelper.java | 161 ++++++++++++++++++ .../apache/tajo/client/ClientParameters.java | 32 ++++ .../apache/tajo/client/QueryClientImpl.java | 14 +- .../apache/tajo/client/SessionConnection.java | 18 +- .../tajo/client/v2/ClientDelegateFactory.java | 10 +- .../tajo/client/v2/LegacyClientDelegate.java | 13 +- .../org/apache/tajo/client/v2/TajoClient.java | 27 +-- .../java/org/apache/tajo/conf/TajoConf.java | 4 + .../tajo/exception/SQLExceptionUtil.java | 4 +- tajo-common/src/main/proto/errors.proto | 4 +- .../tsql/TestDefaultCliOutputFormatter.java | 2 +- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 19 ++- .../tajo/cli/tsql/TestTajoCliNegatives.java | 2 +- .../TestExecExternalShellCommand.java | 2 +- .../cli/tsql/commands/TestHdfsCommand.java | 2 +- .../apache/tajo/util/TestRpcParamFactory.java | 58 +++++++ .../apache/tajo/master/QueryInProgress.java | 12 +- .../org/apache/tajo/master/TajoMaster.java | 7 - .../querymaster/DefaultTaskScheduler.java | 21 ++- .../apache/tajo/querymaster/QueryMaster.java | 20 ++- .../tajo/querymaster/QueryMasterTask.java | 11 +- .../org/apache/tajo/querymaster/Stage.java | 7 +- .../apache/tajo/util/RpcParameterFactory.java | 51 ++++++ .../tajo/worker/ExecutionBlockContext.java | 6 +- .../apache/tajo/worker/NodeStatusUpdater.java | 21 ++- .../org/apache/tajo/worker/TajoWorker.java | 5 - .../org/apache/tajo/worker/TaskManager.java | 12 +- .../ConnectivityCheckerRuleForTajoWorker.java | 4 +- tajo-docs/src/main/sphinx/jdbc_driver.rst | 43 +++++ .../sphinx/table_management/tablespaces.rst | 2 +- .../org/apache/tajo/jdbc/TestTajoJdbc.java | 15 +- .../tajo/jdbc/TestTajoJdbcNegative.java | 54 +++++- .../org/apache/tajo/rpc/RpcConstants.java | 32 +++- .../org/apache/tajo/rpc/AsyncRpcClient.java | 43 +++-- .../apache/tajo/rpc/BlockingRpcClient.java | 45 ++--- .../org/apache/tajo/rpc/NettyClientBase.java | 68 +++++--- .../rpc/ProtoClientChannelInitializer.java | 25 ++- .../org/apache/tajo/rpc/RpcClientManager.java | 131 ++++---------- .../org/apache/tajo/rpc/RpcConnectionKey.java | 56 ++++++ .../org/apache/tajo/rpc/TestAsyncRpc.java | 119 +++++++++---- .../org/apache/tajo/rpc/TestBlockingRpc.java | 115 +++++++++---- .../apache/tajo/rpc/TestRpcClientManager.java | 22 +-- 46 files changed, 1019 insertions(+), 388 deletions(-) create mode 100644 tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java create mode 100644 tajo-client/src/main/java/org/apache/tajo/client/ClientParameterHelper.java create mode 100644 tajo-client/src/main/java/org/apache/tajo/client/ClientParameters.java create mode 100644 tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcParamFactory.java create mode 100644 tajo-core/src/main/java/org/apache/tajo/util/RpcParameterFactory.java create mode 100644 tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcConnectionKey.java diff --git a/CHANGES b/CHANGES index 63af5dd2de..cfad5fe4aa 100644 --- a/CHANGES +++ b/CHANGES @@ -40,6 +40,8 @@ Release 0.11.0 - unreleased IMPROVEMENT + TAJO-1860: Refactor Rpc clients to take Connection Parameters. (hyunsik) + TAJO-1868: Allow TablespaceManager::get to return a unregistered tablespace. (hyunsik) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java index 80ded4a6e1..2b24a6ba9a 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogClient.java @@ -22,16 +22,14 @@ import org.apache.tajo.catalog.CatalogProtocol.CatalogProtocolService.BlockingInterface; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; -import org.apache.tajo.rpc.NettyClientBase; -import org.apache.tajo.rpc.RpcClientManager; -import org.apache.tajo.rpc.RpcConstants; +import org.apache.tajo.rpc.*; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; import org.apache.tajo.util.NetUtils; import java.io.IOException; import java.net.InetSocketAddress; -import java.util.concurrent.TimeUnit; +import java.util.Properties; /** * CatalogClient provides a client API to access the catalog server. @@ -71,10 +69,12 @@ public synchronized NettyClientBase getCatalogConnection() throws ServiceExcepti if (client != null && client.isConnected()) return client; RpcClientManager.cleanup(client); - int retry = conf.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES); + final Properties clientParams = new Properties(); + clientParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, conf.getVar(ConfVars.RPC_CLIENT_RETRY_NUM)); + // Client do not closed on idle state for support high available - this.client = RpcClientManager.getInstance().newClient(getCatalogServerAddr(), CatalogProtocol.class, false, - retry, 0, TimeUnit.SECONDS, false); + this.client = RpcClientManager.getInstance().newClient(getCatalogServerAddr(), CatalogProtocol.class, + false, clientParams); } catch (Exception e) { throw new ServiceException(e); } diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java new file mode 100644 index 0000000000..5fabae8505 --- /dev/null +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/CliClientParamsFactory.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.cli.tsql; + +import org.apache.tajo.annotation.Nullable; +import org.apache.tajo.client.ClientParameters; + +import javax.validation.constraints.Null; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +class CliClientParamsFactory { + static Map DEFAULT_PARAMS = new HashMap<>(); + + static { + // Keep lexicographic order of parameter names. + DEFAULT_PARAMS.put(ClientParameters.CONNECT_TIMEOUT, "0"); + DEFAULT_PARAMS.put(ClientParameters.SOCKET_TIMEOUT, "0"); + DEFAULT_PARAMS.put(ClientParameters.RETRY, "3"); + DEFAULT_PARAMS.put(ClientParameters.ROW_FETCH_SIZE, "200"); + DEFAULT_PARAMS.put(ClientParameters.USE_COMPRESSION, "false"); + } + + public static Properties get(@Nullable Properties connParam) { + Properties copy = connParam == null ? new Properties() : (Properties) connParam.clone(); + for (Map.Entry entry : DEFAULT_PARAMS.entrySet()) { + if (!copy.contains(entry.getKey())) { + copy.setProperty(entry.getKey(), entry.getValue()); + } + } + return copy; + } +} diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java index 83763e8a2b..8ae7075771 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java @@ -18,6 +18,7 @@ package org.apache.tajo.cli.tsql; +import com.google.common.collect.Maps; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.protobuf.ServiceException; @@ -26,6 +27,7 @@ import org.apache.commons.cli.*; import org.apache.tajo.*; import org.apache.tajo.TajoProtos.QueryState; +import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.cli.tsql.ParsedResult.StatementType; import org.apache.tajo.cli.tsql.SimpleParser.ParsingState; @@ -39,16 +41,14 @@ import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.service.ServiceTrackerFactory; import org.apache.tajo.util.FileUtil; +import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.ShutdownHookManager; import java.io.*; import java.lang.reflect.Constructor; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; +import java.util.*; public class TajoCli { public static final int SHUTDOWN_HOOK_PRIORITY = 50; @@ -176,7 +176,9 @@ public Map getCommands() { } } - public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) throws Exception { + public TajoCli(TajoConf c, String [] args, @Nullable Properties clientParams, InputStream in, OutputStream out) + throws Exception { + CommandLineParser parser = new PosixParser(); CommandLine cmd = parser.parse(options, args); @@ -234,14 +236,18 @@ public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) thr } } + // Get connection parameters + Properties defaultConnParams = CliClientParamsFactory.get(clientParams); + final KeyValueSet actualConnParams = new KeyValueSet(Maps.fromProperties(defaultConnParams)); + if ((hostName == null) ^ (port == null)) { System.err.println(ERROR_PREFIX + "cannot find valid Tajo server address"); throw new RuntimeException("cannot find valid Tajo server address"); } else if (hostName != null && port != null) { conf.setVar(ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS, hostName+":"+port); - client = new TajoClientImpl(ServiceTrackerFactory.get(conf), baseDatabase); + client = new TajoClientImpl(ServiceTrackerFactory.get(conf), baseDatabase, actualConnParams); } else if (hostName == null && port == null) { - client = new TajoClientImpl(ServiceTrackerFactory.get(conf), baseDatabase); + client = new TajoClientImpl(ServiceTrackerFactory.get(conf), baseDatabase, actualConnParams); } try { @@ -694,7 +700,7 @@ public void close() { public static void main(String [] args) throws Exception { TajoConf conf = new TajoConf(); - TajoCli shell = new TajoCli(conf, args, System.in, System.out); + TajoCli shell = new TajoCli(conf, args, new Properties(), System.in, System.out); System.out.println(); System.exit(shell.runShell()); } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/ClientParameterHelper.java b/tajo-client/src/main/java/org/apache/tajo/client/ClientParameterHelper.java new file mode 100644 index 0000000000..8549178d2a --- /dev/null +++ b/tajo-client/src/main/java/org/apache/tajo/client/ClientParameterHelper.java @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.client; + +import org.apache.tajo.SessionVars; +import org.apache.tajo.rpc.RpcConstants; +import org.apache.tajo.util.Pair; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import static org.apache.tajo.SessionVars.COMPRESSED_RESULT_TRANSFER; +import static org.apache.tajo.SessionVars.FETCH_ROWNUM; +import static org.apache.tajo.client.ClientParameterHelper.ActionType.CONNECTION_PARAM; +import static org.apache.tajo.client.ClientParameterHelper.ActionType.SESSION_UPDATE; +import static org.apache.tajo.rpc.RpcConstants.CLIENT_CONNECTION_TIMEOUT; +import static org.apache.tajo.rpc.RpcConstants.CLIENT_SOCKET_TIMEOUT; + +/** + *
    + *
  • useCompression=bool - Enable compressed transfer for ResultSet.
    • + *
  • defaultRowFetchSize=int - Determine the number of rows fetched in ResultSet by + * one fetch with trip to the Server.
    • + *
  • connectTimeout=int - The timeout value used for socket connect operations. + * If connecting to the server takes longer than this value,the connection is broken. The + * timeout is specified in seconds and a value of zero means that it is disabled.
    • + *
  • socketTimeout=int
    • - The timeout value used for socket read operations. + * If reading from the server takes longer than this value, the connection is closed. + * This can be used as both a brute force global query timeout and a method of detecting + * network problems. The timeout is specified in seconds and a value of zero means that + * it is disabled. + *
  • retry=intNumber of retry operation. Tajo JDBC driver is resilient + * against some network or connection problems. It determines how many times the connection will retry.
    • + *
+ */ +class ClientParameterHelper { + + public static Map PARAMETERS = new HashMap<>(); + + static { + PARAMETERS.put(ClientParameters.USE_COMPRESSION, new SimpleSessionAction(COMPRESSED_RESULT_TRANSFER)); + PARAMETERS.put(ClientParameters.ROW_FETCH_SIZE, new SimpleSessionAction(FETCH_ROWNUM)); + PARAMETERS.put(ClientParameters.CONNECT_TIMEOUT, new ConnectionParamAction() { + @Override + Pair doAction(String param) { + int seconds = Integer.parseInt(param); + // convert seconds into mili seconds + return new Pair<>(CLIENT_CONNECTION_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(seconds))); + } + }); + PARAMETERS.put(ClientParameters.SOCKET_TIMEOUT, new ConnectionParamAction() { + @Override + Pair doAction(String param) { + int seconds = Integer.parseInt(param); + return new Pair<>(CLIENT_SOCKET_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(seconds))); + } + }); + PARAMETERS.put(ClientParameters.RETRY, new SimpleConnectionParamAction(RpcConstants.CLIENT_RETRY_NUM)); + } + + enum ActionType { + SESSION_UPDATE, + CONNECTION_PARAM + } + + interface Action { + ActionType type(); + } + + static class SimpleSessionAction extends SessionAction { + private final String sessionKey; + + SimpleSessionAction(SessionVars sessionVar) { + this.sessionKey = sessionVar.name(); + } + + Pair doAction(String param) { + return new Pair<>(sessionKey, param); + } + } + + @SuppressWarnings("unused") + static abstract class SessionAction implements Action { + + @Override + public ActionType type() { + return SESSION_UPDATE; + } + + abstract Pair doAction(String param); + } + + static class SimpleConnectionParamAction extends ConnectionParamAction { + final String connParamKey; + + SimpleConnectionParamAction(String connParamKey) { + this.connParamKey = connParamKey; + } + + public Pair doAction(String param) { + return new Pair<>(connParamKey, param); + } + } + + @SuppressWarnings("unused") + static abstract class ConnectionParamAction implements Action { + + @Override + public ActionType type() { + return ActionType.CONNECTION_PARAM; + } + + abstract Pair doAction(String param); + } + + public static Properties getConnParams(Collection> properties) { + Properties connParams = new Properties(); + for (Map.Entry entry : properties) { + if(PARAMETERS.containsKey(entry.getKey()) && PARAMETERS.get(entry.getKey()).type() == CONNECTION_PARAM) { + Pair keyValue = + ((ConnectionParamAction)PARAMETERS.get(entry.getKey())).doAction(entry.getValue()); + connParams.put(keyValue.getFirst(), keyValue.getSecond()); + } + } + + return connParams; + } + + public static Map getSessionVars(Collection> properties) { + Map sessionVars = new HashMap<>(); + + for (Map.Entry entry : properties) { + if(PARAMETERS.containsKey(entry.getKey()) && PARAMETERS.get(entry.getKey()).type() == SESSION_UPDATE) { + Pair keyValue = + ((SessionAction)PARAMETERS.get(entry.getKey())).doAction(entry.getValue()); + sessionVars.put(keyValue.getFirst(), keyValue.getSecond()); + } + } + + return sessionVars; + } +} diff --git a/tajo-client/src/main/java/org/apache/tajo/client/ClientParameters.java b/tajo-client/src/main/java/org/apache/tajo/client/ClientParameters.java new file mode 100644 index 0000000000..e3ee019d46 --- /dev/null +++ b/tajo-client/src/main/java/org/apache/tajo/client/ClientParameters.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.client; + +/** + * Client Parameters which can enable or disable some features of TajoClient. + * This class contains the parameter keys. In more detail, + * please refer to http://tajo.apache.org/docs/current/jdbc_driver.html##connection-parameters + */ +public interface ClientParameters { + String USE_COMPRESSION = "useCompression"; + String ROW_FETCH_SIZE = "defaultRowFetchSize"; + String CONNECT_TIMEOUT = "connectTimeout"; + String SOCKET_TIMEOUT = "socketTimeout"; + String RETRY = "retry"; +} diff --git a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java index 144e3b6ae3..364292e21c 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/QueryClientImpl.java @@ -50,6 +50,7 @@ import java.sql.ResultSet; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -352,7 +353,7 @@ public void run() { protected TajoMemoryResultSet fetchNextQueryResult(final QueryId queryId, final int fetchRowNum) throws TajoException { - boolean compress = conn.getProperties().getBool(SessionVars.COMPRESSED_RESULT_TRANSFER); + final boolean compress = conn.getProperties().getBool(ClientParameters.USE_COMPRESSION); final BlockingInterface stub = conn.getTMStub(); final GetQueryResultDataRequest.Builder request = GetQueryResultDataRequest.newBuilder(); @@ -544,16 +545,7 @@ public QueryHistoryProto getQueryHistory(final QueryId queryId) throws QueryNotF try { - qmClient = manager.newClient( - qmAddress, - QueryMasterClientProtocol.class, - false, - manager.getRetries(), - manager.getTimeoutSeconds(), - TimeUnit.SECONDS, - false - ); - + qmClient = manager.newClient(qmAddress, QueryMasterClientProtocol.class, false, new Properties()); conn.checkSessionAndGet(conn.getTajoMasterConnection()); diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index 3307ade9cd..f63bb47483 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -40,7 +40,6 @@ import org.apache.tajo.rpc.NettyClientBase; import org.apache.tajo.rpc.NettyUtils; import org.apache.tajo.rpc.RpcClientManager; -import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetResponse; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringResponse; @@ -51,10 +50,7 @@ import java.io.Closeable; import java.net.InetSocketAddress; import java.sql.SQLException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.tajo.error.Errors.ResultCode.NO_SUCH_SESSION_VARIABLE; @@ -85,6 +81,8 @@ public class SessionConnection implements Closeable { private NettyClientBase client; + private Properties clientParams; + private final KeyValueSet properties; /** @@ -103,8 +101,9 @@ public SessionConnection(@NotNull ServiceTracker tracker, @Nullable String baseD this.properties = properties; this.manager = RpcClientManager.getInstance(); - this.manager.setRetries(properties.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES)); this.userInfo = UserRoleInfo.getCurrentUser(); + // update the connection parameters to RPC client from connection properties + this.clientParams = ClientParameterHelper.getConnParams(properties.getAllKeyValus().entrySet()); this.eventLoopGroup = NettyUtils.createEventLoopGroup(getClass().getSimpleName(), 4); try { @@ -113,6 +112,9 @@ public SessionConnection(@NotNull ServiceTracker tracker, @Nullable String baseD NettyUtils.shutdown(eventLoopGroup); throw e; } + + // update the session variables from connection parameters + updateSessionVariables(ClientParameterHelper.getSessionVars(properties.getAllKeyValus().entrySet())); } public Map getClientSideSessionVars() { @@ -130,7 +132,7 @@ public synchronized NettyClientBase getTajoMasterConnection() { // Client do not closed on idle state for support high available this.client = manager.newBlockingClient(getTajoMasterAddr(), TajoMasterClientProtocol.class, - manager.getRetries(), eventLoopGroup); + eventLoopGroup, clientParams); } catch (Throwable t) { throw new TajoRuntimeException(new ClientConnectionException(t)); } @@ -217,7 +219,6 @@ public Map updateSessionVariables(final Map vari ensureOk(response.getState()); updateSessionVarsCache(ProtoUtil.convertToMap(response.getSessionVars())); - properties.putAll(sessionVarsCache); return Collections.unmodifiableMap(sessionVarsCache); } @@ -362,7 +363,6 @@ protected void checkSessionAndGet(NettyClientBase client) { } CreateSessionResponse response = null; - try { response = tajoMasterService.createSession(null, builder.build()); } catch (ServiceException se) { diff --git a/tajo-client/src/main/java/org/apache/tajo/client/v2/ClientDelegateFactory.java b/tajo-client/src/main/java/org/apache/tajo/client/v2/ClientDelegateFactory.java index 44721b3c3f..feee37be29 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/v2/ClientDelegateFactory.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/v2/ClientDelegateFactory.java @@ -21,22 +21,22 @@ import org.apache.tajo.annotation.Nullable; import org.apache.tajo.client.v2.exception.ClientUnableToConnectException; -import java.util.Map; +import java.util.Properties; public class ClientDelegateFactory { public static ClientDelegate newDefaultDelegate(String host, int port, - @Nullable Map props) + @Nullable Properties clientParams) throws ClientUnableToConnectException { - return new LegacyClientDelegate(host, port, props); + return new LegacyClientDelegate(host, port, clientParams); } public static ClientDelegate newDefaultDelegate(ServiceDiscovery discovery, - @Nullable Map props) + @Nullable Properties clientParams) throws ClientUnableToConnectException { - return new LegacyClientDelegate(discovery, props); + return new LegacyClientDelegate(discovery, clientParams); } } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/v2/LegacyClientDelegate.java b/tajo-client/src/main/java/org/apache/tajo/client/v2/LegacyClientDelegate.java index 0a2c6defb9..889d974f09 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/v2/LegacyClientDelegate.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/v2/LegacyClientDelegate.java @@ -18,6 +18,7 @@ package org.apache.tajo.client.v2; +import com.google.common.collect.Maps; import com.google.common.util.concurrent.AbstractFuture; import org.apache.tajo.QueryId; import org.apache.tajo.TajoProtos; @@ -40,8 +41,10 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.sql.ResultSet; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -55,13 +58,15 @@ public class LegacyClientDelegate extends SessionConnection implements ClientDel private QueryClientImpl queryClient; private final ExecutorService executor = Executors.newFixedThreadPool(8); - public LegacyClientDelegate(String host, int port, Map props) { - super(new DummyServiceTracker(NetUtils.createSocketAddr(host, port)), null, new KeyValueSet(props)); + public LegacyClientDelegate(String host, int port, Properties clientParams) { + super(new DummyServiceTracker(NetUtils.createSocketAddr(host, port)), null, + new KeyValueSet(clientParams == null ? new HashMap() : Maps.fromProperties(clientParams))); queryClient = new QueryClientImpl(this); } - public LegacyClientDelegate(ServiceDiscovery discovery, Map props) { - super(new DelegateServiceTracker(discovery), null, new KeyValueSet(props)); + public LegacyClientDelegate(ServiceDiscovery discovery, Properties clientParams) { + super(new DelegateServiceTracker(discovery), null, + new KeyValueSet(clientParams == null ? new HashMap() : Maps.fromProperties(clientParams))); queryClient = new QueryClientImpl(this); } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/v2/TajoClient.java b/tajo-client/src/main/java/org/apache/tajo/client/v2/TajoClient.java index 2b4a15042e..dc81742e12 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/v2/TajoClient.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/v2/TajoClient.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.sql.ResultSet; import java.util.Map; +import java.util.Properties; public class TajoClient implements Closeable { private static Log LOG = LogFactory.getLog(TajoClient.class); @@ -52,11 +53,11 @@ public TajoClient(String host) throws ClientUnableToConnectException { /** * Initialize TajoClient with a hostname and default port 26002. * - * @param host Hostname to connect - * @param properties Connection properties + * @param host Hostname to connect + * @param clientParams Client connection parameters */ - public TajoClient(String host, Map properties) throws ClientUnableToConnectException { - delegate = ClientDelegateFactory.newDefaultDelegate(host, DEFAULT_PORT, properties); + public TajoClient(String host, Properties clientParams) throws ClientUnableToConnectException { + delegate = ClientDelegateFactory.newDefaultDelegate(host, DEFAULT_PORT, clientParams); } /** @@ -72,12 +73,12 @@ public TajoClient(String host, int port) throws ClientUnableToConnectException { /** * Initialize TajoClient with a hostname and port * - * @param host Hostname to connect - * @param port Port number to connect - * @param properties Connection properties + * @param host Hostname to connect + * @param port Port number to connect + * @param clientParams Client connection parameters */ - public TajoClient(String host, int port, Map properties) throws ClientUnableToConnectException { - delegate = ClientDelegateFactory.newDefaultDelegate(host, port, properties); + public TajoClient(String host, int port, Properties clientParams) throws ClientUnableToConnectException { + delegate = ClientDelegateFactory.newDefaultDelegate(host, port, clientParams); } /** @@ -92,11 +93,11 @@ public TajoClient(ServiceDiscovery discovery) throws ClientUnableToConnectExcept /** * Initialize TajoClient via service discovery protocol * - * @param discovery Service discovery - * @param properties Connection properties + * @param discovery Service discovery + * @param clientParams Client connection parameters */ - public TajoClient(ServiceDiscovery discovery, Map properties) throws ClientUnableToConnectException { - delegate = ClientDelegateFactory.newDefaultDelegate(discovery, properties); + public TajoClient(ServiceDiscovery discovery, Properties clientParams) throws ClientUnableToConnectException { + delegate = ClientDelegateFactory.newDefaultDelegate(discovery, clientParams); } /** diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 3210323178..d7789f8172 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -230,6 +230,10 @@ public static enum ConfVars implements ConfigKey { // Internal RPC Client INTERNAL_RPC_CLIENT_WORKER_THREAD_NUM("tajo.internal.rpc.client.worker-thread-num", Runtime.getRuntime().availableProcessors() * 2), + RPC_CLIENT_RETRY_NUM("tajo.rpc.client.retry-num", 3, Validators.min("1")), + RPC_CLIENT_CONNECTION_TIMEOUT("tajo.rpc.client.connection-timeout-ms", (long)15 * 1000, Validators.min("0")), + RPC_CLIENT_SOCKET_TIMEOUT("tajo.rpc.client.socket-timeout-ms", (long)180 * 1000, Validators.min("0")), + RPC_CLIENT_HANG_DETECTION_ENABLED("tajo.rpc.client.hang-detection", true, Validators.bool()), // Internal RPC Server MASTER_RPC_SERVER_WORKER_THREAD_NUM("tajo.master.rpc.server.worker-thread-num", diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/SQLExceptionUtil.java b/tajo-common/src/main/java/org/apache/tajo/exception/SQLExceptionUtil.java index 4b4ec97af4..bc40072cb1 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/SQLExceptionUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/SQLExceptionUtil.java @@ -104,9 +104,9 @@ public class SQLExceptionUtil { SQLSTATES.put(ResultCode.INDETERMINATE_DATATYPE, "42P18"); // Client Connection - SQLSTATES.put(ResultCode.CLIENT_CONNECTION_EXCEPTION, "08001"); - SQLSTATES.put(ResultCode.CLIENT_UNABLE_TO_ESTABLISH_CONNECTION, "08002"); + SQLSTATES.put(ResultCode.CLIENT_CONNECTION_EXCEPTION, "08000"); SQLSTATES.put(ResultCode.CLIENT_CONNECTION_DOES_NOT_EXIST, "08003"); + SQLSTATES.put(ResultCode.CLIENT_UNABLE_TO_ESTABLISH_CONNECTION, "08006"); } public static boolean isThisError(SQLException e, ResultCode code) { diff --git a/tajo-common/src/main/proto/errors.proto b/tajo-common/src/main/proto/errors.proto index bb973f2afb..573fc7eb06 100644 --- a/tajo-common/src/main/proto/errors.proto +++ b/tajo-common/src/main/proto/errors.proto @@ -172,9 +172,9 @@ enum ResultCode { // Client Connection CLIENT_CONNECTION_EXCEPTION = 1101; // SQLState: 08000 - Client connection error - CLIENT_UNABLE_TO_ESTABLISH_CONNECTION = 1102; // SQLState: 08001 - CLIENT_CONNECTION_DOES_NOT_EXIST = 1103; // SQLState: 08003 - Client connection has been closed. - CLIENT_PROTOCOL_PROTOCOL_VIOLATION = 1104; // SQLState: ? + CLIENT_UNABLE_TO_ESTABLISH_CONNECTION = 1102; // SQLState: 08006 - Client connection failure + CLIENT_PROTOCOL_PROTOCOL_VIOLATION = 1104; // SQLState: 08P01 - Protocol violation // 53 - Invalid Operand or Inconsistent Specification INSUFFICIENT_RESOURCE = 53000; diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java index 3b53c60a50..cdcdb6762d 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java @@ -66,7 +66,7 @@ public class TestDefaultCliOutputFormatter { public void setUp() throws Exception { conf = cluster.getConfiguration(); ByteArrayOutputStream out = new ByteArrayOutputStream(); - tajoCli = new TajoCli(conf, new String[]{}, System.in, out); + tajoCli = new TajoCli(conf, new String[]{}, null, System.in, out); cliContext = tajoCli.getContext(); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index 8ddef0978f..da51aedd9b 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -29,9 +29,11 @@ import org.apache.tajo.TpchTestBase; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.client.ClientParameters; import org.apache.tajo.client.QueryStatus; import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.FileUtil; @@ -43,6 +45,7 @@ import java.io.*; import java.net.URL; +import java.util.Properties; import static org.junit.Assert.*; @@ -74,7 +77,9 @@ public TestTajoCli() { @Before public void setUp() throws Exception { out = new ByteArrayOutputStream(); - tajoCli = new TajoCli(cluster.getConfiguration(), new String[]{}, System.in, out); + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, "3"); + tajoCli = new TajoCli(cluster.getConfiguration(), new String[]{}, connParams, System.in, out); } @After @@ -155,7 +160,7 @@ public void testParseConf() throws Exception { assertEquals("tajo.executor.join.inner.in-memory-table-num=256", confValues[1]); TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - TajoCli testCli = new TajoCli(tajoConf, args, System.in, System.out); + TajoCli testCli = new TajoCli(tajoConf, args, null, System.in, System.out); try { assertEquals("false", testCli.getContext().get(SessionVars.CLI_PAGING_ENABLED)); assertEquals("256", testCli.getContext().getConf().get("tajo.executor.join.inner.in-memory-table-num")); @@ -321,7 +326,7 @@ public void testGetConf() throws Exception { setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); ByteArrayOutputStream out = new ByteArrayOutputStream(); - TajoCli tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); + TajoCli tajoCli = new TajoCli(tajoConf, new String[]{}, null, System.in, out); try { tajoCli.executeMetaCommand("\\getconf tajo.rootdir"); @@ -338,7 +343,7 @@ public void testShowMasters() throws Exception { setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); ByteArrayOutputStream out = new ByteArrayOutputStream(); - TajoCli tajoCli = new TajoCli(tajoConf, new String[]{}, System.in, out); + TajoCli tajoCli = new TajoCli(tajoConf, new String[]{}, null, System.in, out); tajoCli.executeMetaCommand("\\admin -showmasters"); String consoleResult = new String(out.toByteArray()); @@ -372,7 +377,9 @@ public void run() { TajoConf tajoConf = new TajoConf(); setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); - TajoCli tc = new TajoCli(tajoConf, new String[]{}, is, out); + Properties connParams = new Properties(); + connParams.setProperty(ClientParameters.RETRY, "3"); + TajoCli tc = new TajoCli(tajoConf, new String[]{}, connParams, is, out); tc.executeMetaCommand("\\set ON_ERROR_STOP false"); assertSessionVar(tc, SessionVars.ON_ERROR_STOP.keyname(), "false"); @@ -452,7 +459,7 @@ public void testNonForwardQueryPause() throws Exception { assertEquals(0L, tableDesc.getStats().getNumRows().longValue()); InputStream testInput = new ByteArrayInputStream(new byte[]{(byte) DefaultTajoCliOutputFormatter.QUIT_COMMAND}); - cli = new TajoCli(cluster.getConfiguration(), new String[]{}, testInput, out); + cli = new TajoCli(cluster.getConfiguration(), new String[]{}, null, testInput, out); setVar(cli, SessionVars.CLI_PAGE_ROWS, "2"); setVar(cli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java index 3ebbafa003..d1f1023ffa 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java @@ -40,7 +40,7 @@ public class TestTajoCliNegatives extends QueryTestCaseBase { @BeforeClass public static void setUp() throws Exception { out = new ByteArrayOutputStream(); - tajoCli = new TajoCli(testingCluster.getConfiguration(), new String[]{}, System.in, out); + tajoCli = new TajoCli(testingCluster.getConfiguration(), new String[]{}, null, System.in, out); } @AfterClass diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestExecExternalShellCommand.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestExecExternalShellCommand.java index ed7ee4a7fd..95c3a8bdf8 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestExecExternalShellCommand.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestExecExternalShellCommand.java @@ -34,7 +34,7 @@ public void testCommand() throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); - TajoCli cli = new TajoCli(tajoConf, new String[]{}, null, out); + TajoCli cli = new TajoCli(tajoConf, new String[]{}, null, null, out); cli.executeMetaCommand("\\! echo \"this is test\""); String consoleResult = new String(out.toByteArray()); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestHdfsCommand.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestHdfsCommand.java index 496c7e3f78..d239c0ac00 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestHdfsCommand.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/commands/TestHdfsCommand.java @@ -37,7 +37,7 @@ public void testHdfCommand() throws Exception { System.setOut(new PrintStream(out)); System.setErr(new PrintStream(out)); - TajoCli cli = new TajoCli(tajoConf, new String[]{}, null, out); + TajoCli cli = new TajoCli(tajoConf, new String[]{}, null, null, out); cli.executeMetaCommand("\\dfs -test"); String consoleResult = new String(out.toByteArray()); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcParamFactory.java b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcParamFactory.java new file mode 100644 index 0000000000..1d63bba714 --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestRpcParamFactory.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.util; + +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.rpc.RpcConstants; +import org.junit.Test; + +import java.util.Properties; + +import static org.apache.tajo.rpc.RpcConstants.CLIENT_CONNECTION_TIMEOUT; +import static org.apache.tajo.rpc.RpcConstants.CLIENT_RETRY_NUM; +import static org.junit.Assert.*; + +public class TestRpcParamFactory { + + @Test + public void testGetDefaults() throws Exception { + TajoConf conf = new TajoConf(); + Properties defaultParams = RpcParameterFactory.get(conf); + assertEquals( + ConfVars.RPC_CLIENT_RETRY_NUM.defaultVal, defaultParams.getProperty(CLIENT_RETRY_NUM)); + assertEquals( + ConfVars.RPC_CLIENT_CONNECTION_TIMEOUT.defaultVal, defaultParams.getProperty(CLIENT_CONNECTION_TIMEOUT)); + assertEquals( + ConfVars.RPC_CLIENT_SOCKET_TIMEOUT.defaultVal, defaultParams.getProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT)); + } + + @Test + public void testGet() throws Exception { + TajoConf conf = new TajoConf(); + conf.setIntVar(ConfVars.RPC_CLIENT_RETRY_NUM, 100); + conf.setLongVar(ConfVars.RPC_CLIENT_CONNECTION_TIMEOUT, (long)(10 * 1000)); + conf.setLongVar(ConfVars.RPC_CLIENT_SOCKET_TIMEOUT, (long)60 * 1000); + + Properties defaultParams = RpcParameterFactory.get(conf); + assertEquals("100", defaultParams.getProperty(CLIENT_RETRY_NUM)); + assertEquals(10 * 1000 + "", defaultParams.getProperty(CLIENT_CONNECTION_TIMEOUT)); + assertEquals(60 * 1000 + "", defaultParams.getProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT)); + } +} \ No newline at end of file diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java index b8488768ea..8e999c335b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java @@ -39,9 +39,11 @@ import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.session.Session; import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.RpcParameterFactory; import java.net.ConnectException; import java.net.InetSocketAddress; +import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; @@ -70,6 +72,8 @@ public class QueryInProgress { private AllocationResourceProto allocationResource; + private final Properties rpcParams; + private final Lock readLock; private final Lock writeLock; @@ -87,6 +91,8 @@ public QueryInProgress( queryInfo = new QueryInfo(queryId, queryContext, sql, jsonExpr); queryInfo.setStartTime(System.currentTimeMillis()); + rpcParams = RpcParameterFactory.get(masterContext.getConf()); + ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); @@ -99,7 +105,7 @@ public void kill() { if (queryMasterRpcClient != null) { CallFuture callFuture = new CallFuture(); queryMasterRpcClient.killQuery(callFuture.getController(), queryId.getProto(), callFuture); - callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); } } catch (Throwable e) { catchException("Failed to kill query " + queryId + " by exception " + e, e); @@ -182,7 +188,7 @@ private void connectQueryMaster(WorkerConnectionInfo connectionInfo) InetSocketAddress addr = NetUtils.createSocketAddr(connectionInfo.getHost(), connectionInfo.getQueryMasterPort()); LOG.info("Try to connect to QueryMaster:" + addr); - queryMasterRpc = RpcClientManager.getInstance().newClient(addr, QueryMasterProtocol.class, true); + queryMasterRpc = RpcClientManager.getInstance().newClient(addr, QueryMasterProtocol.class, true, rpcParams); queryMasterRpcClient = queryMasterRpc.getStub(); } @@ -216,7 +222,7 @@ public boolean submitToQueryMaster() { CallFuture callFuture = new CallFuture(); queryMasterRpcClient.executeQuery(callFuture.getController(), builder.build(), callFuture); - callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); querySubmitted = true; getQueryInfo().setQueryState(TajoProtos.QueryState.QUERY_MASTER_LAUNCHED); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index 2f37ceec0f..922a5eb10c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -54,7 +54,6 @@ import org.apache.tajo.metrics.Master; import org.apache.tajo.plan.function.python.PythonScriptEngine; import org.apache.tajo.rpc.RpcClientManager; -import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.rule.EvaluationContext; import org.apache.tajo.rule.EvaluationFailedException; import org.apache.tajo.rule.SelfDiagnosisRuleEngine; @@ -172,14 +171,8 @@ public void serviceInit(Configuration conf) throws Exception { context = new MasterContext(systemConf); clock = new SystemClock(); - RackResolver.init(systemConf); - RpcClientManager rpcManager = RpcClientManager.getInstance(); - rpcManager.setRetries(systemConf.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES)); - rpcManager.setTimeoutSeconds( - systemConf.getInt(RpcConstants.RPC_CLIENT_TIMEOUT_SECS, RpcConstants.DEFAULT_RPC_TIMEOUT_SECONDS)); - initResourceManager(); this.dispatcher = new AsyncDispatcher(); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index b65b5a9dbd..25695a599a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -48,6 +48,7 @@ import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.FetchImpl; @@ -69,6 +70,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler { private final TaskSchedulerContext context; private Stage stage; private TajoConf tajoConf; + private Properties rpcParams; private Thread schedulingThread; private volatile boolean isStopped; @@ -83,7 +85,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler { private int schedulerDelay; private int maximumRequestContainer; - //candidate workers for locality of high priority + // candidate workers for locality of high priority private Set candidateWorkers = Sets.newHashSet(); public DefaultTaskScheduler(TaskSchedulerContext context, Stage stage) { @@ -95,6 +97,8 @@ public DefaultTaskScheduler(TaskSchedulerContext context, Stage stage) { @Override public void init(Configuration conf) { tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class); + rpcParams = RpcParameterFactory.get(new TajoConf()); + scheduledRequests = new ScheduledRequests(); minTaskMemory = tajoConf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY); schedulerDelay= tajoConf.getIntVar(TajoConf.ConfVars.QUERYMASTER_TASK_SCHEDULER_DELAY); @@ -294,7 +298,7 @@ protected LinkedList createTaskRequest(final int incompleteTas ServiceTracker serviceTracker = context.getMasterContext().getQueryMasterContext().getWorkerContext().getServiceTracker(); NettyClientBase tmClient = RpcClientManager.getInstance(). - getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true); + getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true, rpcParams); QueryCoordinatorProtocolService masterClientService = tmClient.getStub(); CallFuture callBack = new CallFuture(); @@ -310,7 +314,7 @@ protected LinkedList createTaskRequest(final int incompleteTas .setQueue(context.getMasterContext().getQueryContext().get("queue", "default")); //TODO set queue masterClientService.reserveNodeResources(callBack.getController(), request.build(), callBack); - NodeResourceResponse response = callBack.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + NodeResourceResponse response = callBack.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); for (AllocationResourceProto resource : response.getResourceList()) { taskRequestEvents.add(new TaskRequestEvent(resource.getWorkerId(), resource, context.getBlockId())); @@ -886,12 +890,14 @@ public void assignToLeafTasks(LinkedList taskRequests) { CallFuture callFuture = new CallFuture(); totalAttempts++; try { - tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true); + tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true, + rpcParams); + TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); tajoWorkerRpcClient.allocateTasks(callFuture.getController(), requestProto.build(), callFuture); BatchAllocationResponse responseProto = - callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); if (responseProto.getCancellationTaskCount() > 0) { for (TaskAllocationProto proto : responseProto.getCancellationTaskList()) { @@ -1004,12 +1010,13 @@ public void assignToNonLeafTasks(LinkedList taskRequests) { AsyncRpcClient tajoWorkerRpc; try { - tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true); + tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true, + rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); tajoWorkerRpcClient.allocateTasks(callFuture.getController(), requestProto.build(), callFuture); BatchAllocationResponse - responseProto = callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + responseProto = callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); if(responseProto.getCancellationTaskCount() > 0) { for (TaskAllocationProto proto : responseProto.getCancellationTaskList()) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index cce94825f3..1b900809ae 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -45,6 +45,7 @@ import org.apache.tajo.rpc.*; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.service.ServiceTracker; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.history.HistoryWriter.WriterFuture; import org.apache.tajo.util.history.HistoryWriter.WriterHolder; @@ -52,10 +53,7 @@ import org.apache.tajo.worker.TajoWorker; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -91,6 +89,8 @@ public class QueryMaster extends CompositeService implements EventHandler { private RpcClientManager manager; + private Properties rpcClientParams; + private ExecutorService eventExecutor; private ExecutorService singleEventExecutor; @@ -105,6 +105,7 @@ public void serviceInit(Configuration conf) throws Exception { this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); this.manager = RpcClientManager.getInstance(); + this.rpcClientParams = RpcParameterFactory.get(this.systemConf); querySessionTimeout = systemConf.getIntVar(TajoConf.ConfVars.QUERY_SESSION_TIMEOUT); queryMasterContext = new QueryMasterContext(systemConf); @@ -171,7 +172,8 @@ public List getAllWorker() { // update master address in worker context. ServiceTracker serviceTracker = workerContext.getServiceTracker(); - rpc = manager.getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true); + rpc = manager.getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true, + rpcClientParams); QueryCoordinatorProtocolService masterService = rpc.getStub(); CallFuture callBack = new CallFuture(); @@ -179,7 +181,7 @@ public List getAllWorker() { PrimitiveProtos.NullProto.getDefaultInstance(), callBack); WorkerConnectionsResponse connectionsProto = - callBack.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callBack.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); return connectionsProto.getWorkerList(); } catch (Exception e) { LOG.error(e.getMessage(), e); @@ -294,11 +296,11 @@ public void stopQuery(final QueryId queryId) { NettyClientBase tmClient; try { tmClient = manager.getClient(workerContext.getServiceTracker().getUmbilicalAddress(), - QueryCoordinatorProtocol.class, true); + QueryCoordinatorProtocol.class, true, rpcClientParams); QueryCoordinatorProtocolService masterClientService = tmClient.getStub(); masterClientService.heartbeat(future.getController(), queryHeartbeat, future); - future.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + future.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); } catch (Exception e) { //this function will be closed in new thread. //When tajo do stop cluster, tajo master maybe throw closed connection exception @@ -404,7 +406,7 @@ public void run() { ServiceTracker serviceTracker = queryMasterContext.getWorkerContext().getServiceTracker(); tmClient = manager.getClient(serviceTracker.getUmbilicalAddress(), - QueryCoordinatorProtocol.class, true); + QueryCoordinatorProtocol.class, true, rpcClientParams); QueryCoordinatorProtocolService masterClientService = tmClient.getStub(); TajoHeartbeatRequest queryHeartbeat = buildTajoHeartBeat(eachTask); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index 46e48e6876..bcfb9388b1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@ -55,6 +55,7 @@ import org.apache.tajo.storage.FormatProperty; import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.event.NodeResourceDeallocateEvent; import org.apache.tajo.worker.event.NodeResourceEvent; @@ -95,6 +96,8 @@ public class QueryMasterTask extends CompositeService { private TajoConf systemConf; + private Properties rpcParams; + private AtomicLong lastClientHeartbeat = new AtomicLong(-1); private volatile boolean isStopped; @@ -131,8 +134,8 @@ public QueryMasterTask(QueryMaster.QueryMasterContext queryMasterContext, @Override public void serviceInit(Configuration conf) throws Exception { - systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); + rpcParams = RpcParameterFactory.get(systemConf); queryTaskContext = new QueryMasterTaskContext(); @@ -255,7 +258,8 @@ protected void killTaskAttempt(int workerId, TaskAttemptId taskAttemptId) { InetSocketAddress workerAddress = getQuery().getStage(ebId).getAssignedWorkerMap().get(workerId); try { - tajoWorkerRpc = RpcClientManager.getInstance().getClient(workerAddress, TajoWorkerProtocol.class, true); + tajoWorkerRpc = RpcClientManager.getInstance().getClient(workerAddress, TajoWorkerProtocol.class, true, + rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); CallFuture callFuture = new CallFuture(); tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), callFuture); @@ -472,7 +476,8 @@ private void cleanupQuery(final QueryId queryId) { @Override public void run() { try { - AsyncRpcClient rpc = RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true); + AsyncRpcClient rpc = RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true, + rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerProtocolService = rpc.getStub(); tajoWorkerProtocolService.stopQuery(null, queryId.getProto(), NullCallback.get()); } catch (Throwable e) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 282edcc81c..98ad2920be 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -62,6 +62,7 @@ import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.KeyValueSet; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.history.StageHistory; import org.apache.tajo.util.history.TaskHistory; @@ -88,6 +89,8 @@ public class Stage implements EventHandler { private static final Log LOG = LogFactory.getLog(Stage.class); + private final Properties rpcParams; + private MasterPlan masterPlan; private ExecutionBlock block; private int priority; @@ -300,6 +303,8 @@ public Stage(QueryMasterTask.QueryMasterTaskContext context, MasterPlan masterPl this.block = block; this.eventHandler = context.getEventHandler(); + this.rpcParams = RpcParameterFactory.get(context.getConf()); + ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); @@ -720,7 +725,7 @@ private void sendStopExecutionBlockEvent(final StopExecutionBlockRequest request public void run() { try { AsyncRpcClient tajoWorkerRpc = - RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true); + RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); tajoWorkerRpcClient.stopExecutionBlock(null, requestProto, NullCallback.get(PrimitiveProtos.BoolProto.class)); diff --git a/tajo-core/src/main/java/org/apache/tajo/util/RpcParameterFactory.java b/tajo-core/src/main/java/org/apache/tajo/util/RpcParameterFactory.java new file mode 100644 index 0000000000..6da4dac459 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/util/RpcParameterFactory.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tajo.util; + +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.rpc.RpcConstants; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +/** + * Helper class to get RPC Client Connection Parameters + */ +public class RpcParameterFactory { + + static final Map PROPERTIES_MAP = new HashMap<>(); + + static { + PROPERTIES_MAP.put(RpcConstants.CLIENT_RETRY_NUM, ConfVars.RPC_CLIENT_RETRY_NUM); + PROPERTIES_MAP.put(RpcConstants.CLIENT_CONNECTION_TIMEOUT, ConfVars.RPC_CLIENT_CONNECTION_TIMEOUT); + PROPERTIES_MAP.put(RpcConstants.CLIENT_SOCKET_TIMEOUT, ConfVars.RPC_CLIENT_SOCKET_TIMEOUT); + PROPERTIES_MAP.put(RpcConstants.CLIENT_HANG_DETECTION, ConfVars.RPC_CLIENT_HANG_DETECTION_ENABLED); + } + + public static Properties get(TajoConf conf) { + final Properties properties = new Properties(); + + for (Map.Entry e : PROPERTIES_MAP.entrySet()) { + properties.put(e.getKey(), conf.getVar(e.getValue())); + } + + return properties; + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java index 94bf785243..a3cc8fc02c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java @@ -253,7 +253,7 @@ public void fatalError(TaskAttemptId taskAttemptId, String message) { //If QueryMaster does not responding, current execution block should be stop CallFuture callFuture = new CallFuture(); getStub().fatalError(callFuture.getController(), builder.build(), callFuture); - callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); } catch (Exception e) { getWorkerContext().getTaskManager().getDispatcher().getEventHandler() .handle(new ExecutionBlockErrorEvent(taskAttemptId.getTaskId().getExecutionBlockId(), e)); @@ -300,7 +300,7 @@ private void sendHashShuffleReport(ExecutionBlockId ebId) throws Exception { CallFuture callFuture = new CallFuture(); stub.doneExecutionBlock(callFuture.getController(), reporterBuilder.build(), callFuture); - callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); return; } @@ -355,7 +355,7 @@ private void sendHashShuffleReport(ExecutionBlockId ebId) throws Exception { try { CallFuture callFuture = new CallFuture(); stub.doneExecutionBlock(callFuture.getController(), reporterBuilder.build(), callFuture); - callFuture.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); } catch (Throwable e) { // can't send report to query master LOG.fatal(e.getMessage(), e); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java index bc4f9a1169..a3b71e1cc0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java @@ -35,12 +35,14 @@ import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.event.NodeStatusEvent; import java.net.ConnectException; import java.util.Collection; import java.util.List; +import java.util.Properties; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -55,7 +57,7 @@ public class NodeStatusUpdater extends AbstractService implements EventHandler heartBeatRequestQueue; private final TajoWorker.WorkerContext workerContext; private AsyncRpcClient rmClient; + private Properties rpcParams; private ServiceTracker serviceTracker; private TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService.Interface resourceTracker; private int queueingThreshold; @@ -75,11 +78,12 @@ public NodeStatusUpdater(TajoWorker.WorkerContext workerContext) { @Override public void serviceInit(Configuration conf) throws Exception { - this.tajoConf = TUtil.checkTypeAndGet(conf, TajoConf.class); + this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); + this.rpcParams = RpcParameterFactory.get(this.systemConf); this.heartBeatRequestQueue = Queues.newLinkedBlockingQueue(); - this.serviceTracker = ServiceTrackerFactory.get(tajoConf); + this.serviceTracker = ServiceTrackerFactory.get(systemConf); this.workerContext.getNodeResourceManager().getDispatcher().register(NodeStatusEvent.EventType.class, this); - this.heartBeatInterval = tajoConf.getIntVar(TajoConf.ConfVars.WORKER_HEARTBEAT_IDLE_INTERVAL); + this.heartBeatInterval = systemConf.getIntVar(TajoConf.ConfVars.WORKER_HEARTBEAT_IDLE_INTERVAL); this.updaterThread = new StatusUpdaterThread(); this.updaterThread.setName("NodeStatusUpdater"); super.serviceInit(conf); @@ -89,10 +93,10 @@ public void serviceInit(Configuration conf) throws Exception { public void serviceStart() throws Exception { DefaultResourceCalculator calculator = new DefaultResourceCalculator(); int maxContainer = calculator.computeAvailableContainers(workerContext.getNodeResourceManager().getTotalResource(), - NodeResources.createResource(tajoConf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY), 1)); + NodeResources.createResource(systemConf.getIntVar(TajoConf.ConfVars.TASK_RESOURCE_MINIMUM_MEMORY), 1)); // if resource changed over than 30%, send reports - float queueingRate = tajoConf.getFloatVar(TajoConf.ConfVars.WORKER_HEARTBEAT_QUEUE_THRESHOLD_RATE); + float queueingRate = systemConf.getFloatVar(TajoConf.ConfVars.WORKER_HEARTBEAT_QUEUE_THRESHOLD_RATE); this.queueingThreshold = Math.max((int) Math.floor(maxContainer * queueingRate), 1); LOG.info("Queueing threshold:" + queueingThreshold); @@ -149,8 +153,7 @@ protected TajoResourceTrackerProtocol.TajoResourceTrackerProtocolService.Interfa RpcClientManager rpcManager = RpcClientManager.getInstance(); rmClient = rpcManager.newClient(serviceTracker.getResourceTrackerAddress(), - TajoResourceTrackerProtocol.class, true, rpcManager.getRetries(), - rpcManager.getTimeoutSeconds(), TimeUnit.SECONDS, false); + TajoResourceTrackerProtocol.class, true, rpcParams); return rmClient.getStub(); } @@ -165,7 +168,7 @@ protected NodeHeartbeatResponse sendHeartbeat(NodeHeartbeatRequest requestProto) CallFuture callBack = new CallFuture(); resourceTracker.nodeHeartbeat(callBack.getController(), requestProto, callBack); - response = callBack.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + response = callBack.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); } catch (InterruptedException e) { LOG.warn(e.getMessage()); } catch (TimeoutException te) { diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java index fbb8d5426c..607e7ff05b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java @@ -152,11 +152,6 @@ public void serviceInit(Configuration conf) throws Exception { this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class); RackResolver.init(systemConf); - RpcClientManager rpcManager = RpcClientManager.getInstance(); - rpcManager.setRetries(systemConf.getInt(RpcConstants.RPC_CLIENT_RETRY_MAX, RpcConstants.DEFAULT_RPC_RETRIES)); - rpcManager.setTimeoutSeconds( - systemConf.getInt(RpcConstants.RPC_CLIENT_TIMEOUT_SECS, RpcConstants.DEFAULT_RPC_TIMEOUT_SECONDS)); - serviceTracker = ServiceTrackerFactory.get(systemConf); this.workerContext = new TajoWorkerContext(); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java index 42db852dc1..a0b3f97f17 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java @@ -35,6 +35,7 @@ import org.apache.tajo.rpc.RpcClientManager; import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.event.*; @@ -42,11 +43,10 @@ import java.net.InetSocketAddress; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.TimeUnit; -import static org.apache.tajo.ResourceProtos.ExecutionBlockListProto; -import static org.apache.tajo.ResourceProtos.ExecutionBlockContextRequest; -import static org.apache.tajo.ResourceProtos.ExecutionBlockContextResponse; +import static org.apache.tajo.ResourceProtos.*; /** * A TaskManager is responsible for managing executionBlock resource and tasks. @@ -58,6 +58,7 @@ public class TaskManager extends AbstractService implements EventHandler executionBlockContextMap; private final Dispatcher dispatcher; private TaskExecutor executor; + private final Properties rpcParams; public TaskManager(Dispatcher dispatcher, TajoWorker.WorkerContext workerContext){ this(dispatcher, workerContext, null); @@ -70,6 +71,7 @@ public TaskManager(Dispatcher dispatcher, TajoWorker.WorkerContext workerContext this.workerContext = workerContext; this.executionBlockContextMap = Maps.newHashMap(); this.executor = executor; + this.rpcParams = RpcParameterFactory.get(this.workerContext.getConf()); } @Override @@ -118,13 +120,13 @@ protected ExecutionBlockContext createExecutionBlock(ExecutionBlockId executionB request.setExecutionBlockId(executionBlockId.getProto()) .setWorker(getWorkerContext().getConnectionInfo().getProto()); - client = RpcClientManager.getInstance().newClient(address, QueryMasterProtocol.class, true); + client = RpcClientManager.getInstance().newClient(address, QueryMasterProtocol.class, true, rpcParams); QueryMasterProtocol.QueryMasterProtocolService.Interface stub = client.getStub(); CallFuture callback = new CallFuture(); stub.getExecutionBlockContext(callback.getController(), request.build(), callback); ExecutionBlockContextResponse contextProto = - callback.get(RpcConstants.DEFAULT_FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + callback.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); ExecutionBlockContext context = new ExecutionBlockContext(getWorkerContext(), contextProto, client); context.init(); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java index f94bd78d23..ee428cd6f5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java @@ -26,6 +26,7 @@ import org.apache.tajo.rule.EvaluationResult.EvaluationResultCode; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; +import org.apache.tajo.util.RpcParameterFactory; import org.apache.tajo.worker.TajoWorker; /** @@ -40,7 +41,8 @@ private void checkTajoMasterConnectivity(TajoConf tajoConf) throws Exception { RpcClientManager manager = RpcClientManager.getInstance(); ServiceTracker serviceTracker = ServiceTrackerFactory.get(tajoConf); - NettyClientBase masterClient = manager.getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true); + NettyClientBase masterClient = manager.getClient(serviceTracker.getUmbilicalAddress(), + QueryCoordinatorProtocol.class, true, RpcParameterFactory.get(tajoConf)); masterClient.getStub(); } diff --git a/tajo-docs/src/main/sphinx/jdbc_driver.rst b/tajo-docs/src/main/sphinx/jdbc_driver.rst index 6c7371bf4e..176477aba9 100644 --- a/tajo-docs/src/main/sphinx/jdbc_driver.rst +++ b/tajo-docs/src/main/sphinx/jdbc_driver.rst @@ -42,6 +42,49 @@ In order to use the JDBC driver, you should add ``tajo-jdbc-x.y.z.jar`` in your CLASSPATH=path/to/tajo-jdbc-x.y.z.jar:$CLASSPATH +Connecting to the Tajo cluster instance +======================================= +A Tajo cluster is represented by a URL. Tajo JDBC driver can take the following URL forms: + + * ``jdbc:tajo://host/`` + * ``jdbc:tajo://host/database`` + * ``jdbc:tajo://host:port/`` + * ``jdbc:tajo://host:port/database`` + +Each part of URL has the following meanings: + + * ``host`` - The hostname of the TajoMaster. You can put hostname or ip address here. + * ``port`` - The port number that server is listening. Default port number is 26002. + * ``database`` - The database name. The default database name is ``default``. + + To connect, you need to get ``Connection`` instance from Java JDBC Driver Manager as follows: + +.. code-block:: java + + Connection db = DriverManager.getConnection(url); + + +Connection Parameters +===================== +Connection parameters lets the JDBC Copnnection to enable or disable additional features. You should use ``java.util.Properties`` to pass your connection parameters into ``Connection``. The following example means that the transmission of ResultSet uses compression and its connection timeout is 15 seconds. + +.. code-block:: java + + String url = "jdbc:tajo://localhost/test"; + Properties props = new Properties(); + props.setProperty("useCompression","true"); // use compression for ResultSet + props.setProperty("connectTimeout","15000"); // 15 seconds + Connection conn = DriverManager.getConnection(url, props); + +The connection parameters that Tajo currently supports are as follows: + + * ``useCompression = bool`` - Enable compressed transfer for ResultSet. + * ``defaultRowFetchSize = int`` - Determine the number of rows fetched in ResultSet by one fetch with trip to the Server. + * ``connectTimeout = int (seconds)`` - The timeout value used for socket connect operations. If connecting to the server takes longer than this value, the connection is broken. The timeout is specified in seconds and a value of zero means that it is disabled. + * ``socketTimeout = int (seconds)`` - The timeout value used for socket read operations. If reading from the server takes longer than this value, the connection is closed. This can be used as both a brute force global query timeout and a method of detecting network problems. The timeout is specified in seconds and a value of zero means that it is disabled. + * ``retry = int`` - Number of retry operation. Tajo JDBC driver is resilient against some network or connection problems. It determines how many times the connection will retry. + + An Example JDBC Client ======================= diff --git a/tajo-docs/src/main/sphinx/table_management/tablespaces.rst b/tajo-docs/src/main/sphinx/table_management/tablespaces.rst index 964491c7b9..79ea65f1f5 100644 --- a/tajo-docs/src/main/sphinx/table_management/tablespaces.rst +++ b/tajo-docs/src/main/sphinx/table_management/tablespaces.rst @@ -42,4 +42,4 @@ The following is an example for two tablespaces for hbase and hdfs: .. note:: - Also, each tablespace can use different storage type. Please see :doc:`/storage_plugin` if you want to know more information about it. \ No newline at end of file + Also, each tablespace can use different storage type. Please see :doc:`/storage_plugins` if you want to know more information about it. \ No newline at end of file diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index a860d5119e..f8d5c4576b 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -19,21 +19,32 @@ package org.apache.tajo.jdbc; import com.google.common.collect.Maps; +import io.netty.channel.ConnectTimeoutException; import org.apache.tajo.*; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.client.QueryStatus; +import org.apache.tajo.error.Errors; +import org.apache.tajo.exception.SQLExceptionUtil; +import org.apache.tajo.util.UriUtil; import org.junit.AfterClass; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import java.io.IOException; import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; import java.sql.*; import java.util.*; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; +import static org.apache.tajo.error.Errors.ResultCode.CLIENT_CONNECTION_EXCEPTION; +import static org.apache.tajo.error.Errors.ResultCode.CLIENT_UNABLE_TO_ESTABLISH_CONNECTION; import static org.junit.Assert.*; @Category(IntegrationTest.class) @@ -197,7 +208,7 @@ public void testPreparedStatement() throws Exception { public void testResultSetCompression() throws Exception { String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(), TajoConstants.DEFAULT_DATABASE_NAME); - connUri = connUri + "?" + SessionVars.COMPRESSED_RESULT_TRANSFER.keyname() + "=true"; + connUri = connUri + "?useCompression=true"; Connection conn = DriverManager.getConnection(connUri); assertTrue(conn.isValid(100)); @@ -593,7 +604,7 @@ public void testAlterTableAddPartition() throws Exception { try { if (!testingCluster.isHiveCatalogStoreRunning()) { String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), - tajoMasterAddress.getPort(), "TestTajoJdbc"); + tajoMasterAddress.getPort(), "TestTajoJdbc"); conn = DriverManager.getConnection(connUri); assertTrue(conn.isValid(100)); diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbcNegative.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbcNegative.java index 8f84226c24..1050fc0c61 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbcNegative.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbcNegative.java @@ -21,7 +21,10 @@ import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.error.Errors.ResultCode; +import org.apache.tajo.exception.SQLExceptionUtil; +import org.apache.tajo.util.UriUtil; import org.junit.AfterClass; +import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -29,9 +32,13 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; import java.sql.*; +import java.util.Properties; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; +import static org.apache.tajo.error.Errors.ResultCode.CLIENT_CONNECTION_EXCEPTION; import static org.apache.tajo.exception.SQLExceptionUtil.toSQLState; import static org.apache.tajo.jdbc.TestTajoJdbc.buildConnectionUri; import static org.junit.Assert.*; @@ -53,7 +60,7 @@ public static void tearDown() throws Exception { @Test(expected = SQLException.class) public void testGetConnection() throws SQLException { DriverManager.getConnection("jdbc:taju://" + tajoMasterAddress.getHostName() + ":" + tajoMasterAddress.getPort() - + "/default"); + + "/default"); } @Test @@ -191,4 +198,49 @@ public void testExceptionDuringProcessing() throws Exception { } } } + + private void assumeConnectTimeout(String host, int port, int connectTimeout) throws IOException { + try (Socket socket = new Socket()) { + // Try to connect to a private address in the 10.x.y.z range. + // These addresses are usually not routed, so an attempt to + // connect to them will hang the connection attempt, which is + // what we want to simulate in this test. + socket.connect(new InetSocketAddress(host, port), connectTimeout); + // Abort the test if we can connect. + Assume.assumeTrue(false); + } catch (SocketTimeoutException x) { + // Expected timeout during connect, continue the test. + Assume.assumeTrue(true); + } catch (Throwable x) { + // Abort if any other exception happens. + Assume.assumeTrue(false); + } + } + + @Test(timeout = 5000) + public final void testConnectTimeout() throws Exception { + final String host = "10.255.255.1"; + final int port = 80; + int connectTimeout = 1000; + assumeConnectTimeout(host, port, connectTimeout); + + long startTime = Long.MIN_VALUE; + long endTime; + try { + // artificially cause connection timeout + String connUri = buildConnectionUri(host, port, DEFAULT_DATABASE_NAME); + connUri = UriUtil.addParam(connUri, "connectTimeout", "1"); // 1 seconds + connUri = UriUtil.addParam(connUri, "retry", "0"); // 1 seconds + startTime = System.currentTimeMillis(); + new JdbcConnection(connUri, new Properties()); + fail("Must be failed"); + } catch (SQLException t) { + endTime = System.currentTimeMillis(); + assertEquals(t.getSQLState(), SQLExceptionUtil.toSQLState(CLIENT_CONNECTION_EXCEPTION)); + assertEquals("connection timed out: /10.255.255.1:80", t.getMessage()); + // default is 15 seconds. So, if timeout is shorter than 1~2 seconds. + // We can ensure the parameter was effective. + assertTrue(((endTime - startTime) / 1000) < 2); + } + } } diff --git a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java index ab0826fbfb..601f3d2156 100644 --- a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java +++ b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/RpcConstants.java @@ -18,15 +18,33 @@ package org.apache.tajo.rpc; +import java.util.concurrent.TimeUnit; + +/** + * Constants for RPC + */ public class RpcConstants { public static final String PING_PACKET = "TAJO"; - public static final String RPC_CLIENT_RETRY_MAX = "tajo.rpc.client.retry.max"; - public static final String RPC_CLIENT_TIMEOUT_SECS = "tajo.rpc.client.timeout-secs"; - - public static final int DEFAULT_RPC_RETRIES = 3; - public static final int DEFAULT_RPC_TIMEOUT_SECONDS = 180; - public static final int DEFAULT_CONNECT_TIMEOUT = 20000; // 20 sec public static final int DEFAULT_PAUSE = 1000; // 1 sec - public static final int DEFAULT_FUTURE_TIMEOUT_SECONDS = 10; + public static final int FUTURE_TIMEOUT_SECONDS_DEFAULT = 10; + + /** How many times the connect will retry */ + public static final String CLIENT_RETRY_NUM = "tajo.rpc.client.retry-num"; + public static final int CLIENT_RETRY_NUM_DEFAULT = 0; + + /** Client connection timeout (milliseconds) */ + public static final String CLIENT_CONNECTION_TIMEOUT = "tajo.rpc.client.connection-timeout-ms"; + /** Default client connection timeout 15 seconds */ + public final static long CLIENT_CONNECTION_TIMEOUT_DEFAULT = TimeUnit.SECONDS.toMillis(15); + + /** + * Socket timeout (milliseconds). + */ + public static final String CLIENT_SOCKET_TIMEOUT = "tajo.rpc.client.socket-timeout-ms"; + /** Default socket timeout - 60 seconds */ + public final static long CLIENT_SOCKET_TIMEOUT_DEFAULT = TimeUnit.SECONDS.toMillis(180); + + public static final String CLIENT_HANG_DETECTION = "tajo.rpc.client.hang-detection"; + public final static boolean CLIENT_HANG_DETECTION_DEFAULT = false; } diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java index 6fb62d4743..c613bace99 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java @@ -18,56 +18,55 @@ package org.apache.tajo.rpc; -import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.Descriptors.MethodDescriptor; import com.google.protobuf.*; import io.netty.channel.ChannelHandler; import io.netty.channel.EventLoopGroup; -import org.apache.tajo.rpc.RpcClientManager.RpcConnectionKey; import org.apache.tajo.rpc.RpcProtos.RpcResponse; import java.lang.reflect.Method; -import java.util.concurrent.TimeUnit; +import java.util.Properties; import java.util.concurrent.atomic.AtomicInteger; +import static org.apache.tajo.rpc.RpcConstants.*; + public class AsyncRpcClient extends NettyClientBase { private final Method stubMethod; private final ProxyRpcChannel rpcChannel; private final NettyChannelInboundHandler handler; - @VisibleForTesting - AsyncRpcClient(RpcConnectionKey rpcConnectionKey, int retries) - throws ClassNotFoundException, NoSuchMethodException { - this(rpcConnectionKey, retries, 0, TimeUnit.NANOSECONDS, false, NettyUtils.getDefaultEventLoopGroup()); - } /** * Intentionally make this method package-private, avoiding user directly * new an instance through this constructor. * - * @param rpcConnectionKey - * @param retries retry operation number of times - * @param timeout disable ping, it trigger timeout event on idle-state. - * otherwise it is request timeout on active-state - * @param timeUnit TimeUnit - * @param enablePing enable to detect remote peer hangs - * @param eventLoopGroup thread pool of netty's + * @param rpcConnectionKey RpcConnectionKey + * @param eventLoopGroup Thread pool of netty's + * @param rpcParams Rpc connection parameters (see RpcConstants) + * * @throws ClassNotFoundException * @throws NoSuchMethodException */ - AsyncRpcClient(RpcConnectionKey rpcConnectionKey, int retries, long timeout, TimeUnit timeUnit, boolean enablePing, - EventLoopGroup eventLoopGroup) + AsyncRpcClient(EventLoopGroup eventLoopGroup, + RpcConnectionKey rpcConnectionKey, + Properties rpcParams) throws ClassNotFoundException, NoSuchMethodException { - super(rpcConnectionKey, retries); + super(rpcConnectionKey, rpcParams); this.stubMethod = getServiceClass().getMethod("newStub", RpcChannel.class); this.rpcChannel = new ProxyRpcChannel(); this.handler = new ClientChannelInboundHandler(); - init(new ProtoClientChannelInitializer(handler, - RpcResponse.getDefaultInstance(), - timeUnit.toNanos(timeout), - enablePing), eventLoopGroup); + + final long socketTimeoutMills = Long.parseLong( + rpcParams.getProperty(CLIENT_SOCKET_TIMEOUT, String.valueOf(CLIENT_SOCKET_TIMEOUT_DEFAULT))); + + // Enable proactive hang detection + final boolean hangDetectionEnabled = Boolean.parseBoolean( + rpcParams.getProperty(CLIENT_HANG_DETECTION, String.valueOf(CLIENT_HANG_DETECTION_DEFAULT))); + + init(new ProtoClientChannelInitializer(handler, RpcResponse.getDefaultInstance(), socketTimeoutMills, + hangDetectionEnabled), eventLoopGroup); } @Override diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java index 4327003b67..35675b4bb7 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java @@ -18,56 +18,57 @@ package org.apache.tajo.rpc; -import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.*; import com.google.protobuf.Descriptors.MethodDescriptor; import io.netty.channel.ChannelHandler; import io.netty.channel.EventLoopGroup; -import org.apache.tajo.rpc.RpcClientManager.RpcConnectionKey; import org.apache.tajo.rpc.RpcProtos.RpcResponse; import java.lang.reflect.Method; import java.net.InetSocketAddress; +import java.util.Properties; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; +import static org.apache.tajo.rpc.RpcConstants.*; + public class BlockingRpcClient extends NettyClientBase { private final Method stubMethod; private final ProxyRpcChannel rpcChannel; private final NettyChannelInboundHandler handler; - @VisibleForTesting - BlockingRpcClient(RpcConnectionKey rpcConnectionKey, int retries) - throws NoSuchMethodException, ClassNotFoundException { - this(rpcConnectionKey, retries, 0, TimeUnit.NANOSECONDS, false, NettyUtils.getDefaultEventLoopGroup()); - } - /** * Intentionally make this method package-private, avoiding user directly * new an instance through this constructor. * - * @param rpcConnectionKey - * @param retries retry operation number of times - * @param timeout disable ping, it trigger timeout event on idle-state. - * otherwise it is request timeout on active-state - * @param timeUnit TimeUnit - * @param enablePing enable to detect remote peer hangs - * @param eventLoopGroup thread pool of netty's + * @param rpcConnectionKey RpcConnectionKey + * @param eventLoopGroup Thread pool of netty's + * @param rpcParams Rpc connection parameters (see RpcConstants) + * * @throws ClassNotFoundException * @throws NoSuchMethodException + * @see RpcConstants */ - BlockingRpcClient(RpcConnectionKey rpcConnectionKey, int retries, long timeout, TimeUnit timeUnit, boolean enablePing, - EventLoopGroup eventLoopGroup) throws ClassNotFoundException, NoSuchMethodException { - super(rpcConnectionKey, retries); + public BlockingRpcClient(EventLoopGroup eventLoopGroup, + RpcConnectionKey rpcConnectionKey, + Properties rpcParams) + throws ClassNotFoundException, NoSuchMethodException { + super(rpcConnectionKey, rpcParams); this.stubMethod = getServiceClass().getMethod("newBlockingStub", BlockingRpcChannel.class); this.rpcChannel = new ProxyRpcChannel(); this.handler = new ClientChannelInboundHandler(); - init(new ProtoClientChannelInitializer(handler, - RpcResponse.getDefaultInstance(), - timeUnit.toNanos(timeout), - enablePing), eventLoopGroup); + + long socketTimeoutMills = Long.parseLong( + rpcParams.getProperty(CLIENT_SOCKET_TIMEOUT, String.valueOf(CLIENT_SOCKET_TIMEOUT_DEFAULT))); + + // Enable proactive hang detection + final boolean hangDetectionEnabled = Boolean.parseBoolean( + rpcParams.getProperty(CLIENT_HANG_DETECTION, String.valueOf(CLIENT_HANG_DETECTION_DEFAULT))); + + init(new ProtoClientChannelInitializer(handler, RpcResponse.getDefaultInstance(), socketTimeoutMills, + hangDetectionEnabled), eventLoopGroup); } @Override diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java index e5485da74f..6008c4c709 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/NettyClientBase.java @@ -18,6 +18,7 @@ package org.apache.tajo.rpc; +import com.google.common.base.Preconditions; import com.google.protobuf.Descriptors; import com.google.protobuf.Message; import com.google.protobuf.ServiceException; @@ -31,7 +32,6 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.tajo.rpc.RpcClientManager.RpcConnectionKey; import org.apache.tajo.rpc.RpcProtos.RpcResponse; import java.io.Closeable; @@ -41,27 +41,50 @@ import java.net.SocketAddress; import java.nio.channels.UnresolvedAddressException; import java.util.Collection; +import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; +import static org.apache.tajo.rpc.RpcConstants.*; + public abstract class NettyClientBase implements ProtoDeclaration, Closeable { public final static Log LOG = LogFactory.getLog(NettyClientBase.class); - private Bootstrap bootstrap; - private volatile ChannelFuture channelFuture; private final RpcConnectionKey key; - private final int maxRetries; + /** Number to retry for connection and RPC invocation */ + private final int maxRetryNum; + /** Connection Timeout */ + private final long connTimeoutMillis; private boolean enableMonitor; - - private final ConcurrentMap channelEventListeners = - new ConcurrentHashMap<>(); + private final ConcurrentMap channelEventListeners = new ConcurrentHashMap<>(); private final ConcurrentMap requests = new ConcurrentHashMap<>(); - public NettyClientBase(RpcConnectionKey rpcConnectionKey, int numRetries) + private Bootstrap bootstrap; + private volatile ChannelFuture channelFuture; + + /** + * Constructor of NettyClientBase + * + * @param rpcConnectionKey RpcConnectionKey + * @param rpcParams Rpc connection parameters (see RpcConstants) + * + * @throws ClassNotFoundException + * @throws NoSuchMethodException + * @see RpcConstants + */ + public NettyClientBase(RpcConnectionKey rpcConnectionKey, Properties rpcParams) throws ClassNotFoundException, NoSuchMethodException { this.key = rpcConnectionKey; - this.maxRetries = numRetries; + + this.maxRetryNum = Integer.parseInt( + rpcParams.getProperty(CLIENT_RETRY_NUM, String.valueOf(CLIENT_RETRY_NUM_DEFAULT))); + + this.connTimeoutMillis = Integer.parseInt( + rpcParams.getProperty(CLIENT_CONNECTION_TIMEOUT, String.valueOf(CLIENT_CONNECTION_TIMEOUT_DEFAULT))); + + // Netty only takes integer value range and this is to avoid integer overflow. + Preconditions.checkArgument(this.connTimeoutMillis <= Integer.MAX_VALUE, "Too long connection timeout"); } // should be called from sub class @@ -73,12 +96,12 @@ protected void init(ChannelInitializer initializer, EventLoopGroup even .handler(initializer) .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) .option(ChannelOption.SO_REUSEADDR, true) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, RpcConstants.DEFAULT_CONNECT_TIMEOUT) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, (int) connTimeoutMillis) .option(ChannelOption.SO_RCVBUF, 1048576 * 10) .option(ChannelOption.TCP_NODELAY, true); } - public RpcClientManager.RpcConnectionKey getKey() { + public RpcConnectionKey getKey() { return key; } @@ -132,7 +155,7 @@ public void operationComplete(final ChannelFuture future) throws Exception { getHandler().registerCallback(rpcRequest.getId(), callback); } else { - if (!future.channel().isActive() && retry < maxRetries) { + if (!future.channel().isActive() && retry < maxRetryNum) { /* schedule the current request for the retry */ LOG.warn(future.cause() + " Try to reconnect :" + getKey().addr); @@ -173,6 +196,14 @@ private ChannelFuture doConnect(SocketAddress address) { return this.channelFuture = bootstrap.clone().connect(address); } + private ConnectException makeConnectException(InetSocketAddress address, ChannelFuture future) { + if (future.cause() instanceof UnresolvedAddressException) { + return new ConnectException("Can't resolve host name: " + address.toString()); + } else { + return new ConnectTimeoutException(future.cause().getMessage()); + } + } + public synchronized void connect() throws ConnectException { if (isConnected()) return; @@ -186,10 +217,10 @@ public synchronized void connect() throws ConnectException { ChannelFuture f = doConnect(address).awaitUninterruptibly(); if (!f.isSuccess()) { - if (maxRetries > 0) { + if (maxRetryNum > 0) { doReconnect(address, f, ++retries); } else { - throw new ConnectException(ExceptionUtils.getMessage(f.cause())); + throw makeConnectException(address, f); } } } @@ -198,7 +229,7 @@ private void doReconnect(final InetSocketAddress address, ChannelFuture future, throws ConnectException { for (; ; ) { - if (maxRetries > retries) { + if (maxRetryNum > retries) { retries++; if(getChannel().eventLoop().isShuttingDown()) { @@ -218,12 +249,7 @@ private void doReconnect(final InetSocketAddress address, ChannelFuture future, } } else { LOG.error("Max retry count has been exceeded. attempts=" + retries + " caused by: " + future.cause()); - - if (future.cause() instanceof UnresolvedAddressException) { - throw new ConnectException("Can't resolve host name: " + address.toString()); - } else { - throw new ConnectTimeoutException(future.cause().getMessage()); - } + throw makeConnectException(address, future); } } } diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java index 8787dee3cb..5d544cb9ef 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/ProtoClientChannelInitializer.java @@ -34,26 +34,33 @@ class ProtoClientChannelInitializer extends ChannelInitializer { private final MessageLite defaultInstance; private final ChannelHandler handler; - private final long timeoutTimeNanos; - private final boolean enablePing; + private final long idleTimeout; + private final boolean hangDetection; + /** + * Channel Pipe Initializer + * + * @param handler Channel Handler + * @param defaultInstance Default Rpc Proto instance + * @param idleTimeout Idle timeout (milliseconds) + */ public ProtoClientChannelInitializer(ChannelHandler handler, MessageLite defaultInstance, - long timeoutTimeNanos, - boolean enablePing) { + long idleTimeout, boolean hangDetection) { this.handler = handler; this.defaultInstance = defaultInstance; - this.timeoutTimeNanos = timeoutTimeNanos; - this.enablePing = enablePing; + this.idleTimeout = idleTimeout; + this.hangDetection = hangDetection; } @Override protected void initChannel(Channel channel) throws Exception { ChannelPipeline pipeline = channel.pipeline(); pipeline.addLast("idleStateHandler", - new IdleStateHandler(timeoutTimeNanos, timeoutTimeNanos / 2, 0, TimeUnit.NANOSECONDS)); - - if (enablePing) pipeline.addLast("MonitorClientHandler", new MonitorClientHandler()); + new IdleStateHandler(idleTimeout, idleTimeout / 2, 0, TimeUnit.MILLISECONDS)); + if (hangDetection) { + pipeline.addLast("MonitorClientHandler", new MonitorClientHandler()); + } pipeline.addLast("frameDecoder", new ProtobufVarint32FrameDecoder()); pipeline.addLast("protobufDecoder", new ProtobufDecoder(defaultInstance)); pipeline.addLast("frameEncoder", new ProtobufVarint32LengthFieldPrepender()); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java index c801b8a209..032cf355e8 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcClientManager.java @@ -33,15 +33,12 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.TimeUnit; +import java.util.Properties; @ThreadSafe public class RpcClientManager { private static final Log LOG = LogFactory.getLog(RpcClientManager.class); - private volatile int timeoutSeconds = RpcConstants.DEFAULT_RPC_TIMEOUT_SECONDS; - private volatile int retries = RpcConstants.DEFAULT_RPC_RETRIES; - /* entries will be removed by ConnectionCloseFutureListener */ private static final Map clients = Collections.synchronizedMap(new HashMap()); @@ -61,26 +58,23 @@ public static RpcClientManager getInstance() { } private T makeClient(RpcConnectionKey rpcConnectionKey, - int retries, - long timeout, - TimeUnit timeUnit, - boolean enablePing) + Properties rpcParams) throws NoSuchMethodException, ConnectException, ClassNotFoundException { - return makeClient(rpcConnectionKey, retries, timeout, timeUnit, enablePing, NettyUtils.getDefaultEventLoopGroup()); + + + return makeClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, rpcParams); } - private T makeClient(RpcConnectionKey rpcConnectionKey, - int retries, - long timeout, - TimeUnit timeUnit, - boolean enablePing, - EventLoopGroup eventLoopGroup) + private T makeClient(EventLoopGroup eventLoopGroup, + RpcConnectionKey rpcConnectionKey, + Properties rpcParams) throws NoSuchMethodException, ClassNotFoundException, ConnectException { NettyClientBase client; if (rpcConnectionKey.asyncMode) { - client = new AsyncRpcClient(rpcConnectionKey, retries, timeout, timeUnit, enablePing, eventLoopGroup); + client = new AsyncRpcClient(eventLoopGroup, rpcConnectionKey, rpcParams); + } else { - client = new BlockingRpcClient(rpcConnectionKey, retries, timeout, timeUnit, enablePing, eventLoopGroup); + client = new BlockingRpcClient(eventLoopGroup, rpcConnectionKey, rpcParams); } return (T) client; } @@ -90,7 +84,9 @@ private T makeClient(RpcConnectionKey rpcConnectionK * This client will be shared per protocol and address. Client is removed in shared map when a client is closed */ public T getClient(InetSocketAddress addr, - Class protocolClass, boolean asyncMode) + Class protocolClass, + boolean asyncMode, + Properties rpcParams) throws NoSuchMethodException, ClassNotFoundException, ConnectException { RpcConnectionKey key = new RpcConnectionKey(addr, protocolClass, asyncMode); @@ -98,7 +94,7 @@ public T getClient(InetSocketAddress addr, synchronized (clients) { client = clients.get(key); if (client == null) { - clients.put(key, client = makeClient(key, retries, getTimeoutSeconds(), TimeUnit.SECONDS, true)); + clients.put(key, client = makeClient(key, rpcParams)); } } @@ -129,35 +125,30 @@ public void channelUnregistered(ChannelHandlerContext ctx) { * Connect a {@link NettyClientBase} to the remote {@link NettyServerBase}, and returns rpc client by protocol. * This client does not managed. It should close. */ - public synchronized T newClient(InetSocketAddress addr, + public T newClient(InetSocketAddress addr, Class protocolClass, boolean asyncMode, - int retries, - long timeout, - TimeUnit timeUnit, - boolean enablePing) - throws NoSuchMethodException, ClassNotFoundException, ConnectException { - - return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), retries, timeout, timeUnit, enablePing); - } - - public synchronized T newClient(InetSocketAddress addr, - Class protocolClass, - boolean asyncMode) + Properties rpcParams) throws NoSuchMethodException, ClassNotFoundException, ConnectException { - return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), - retries, getTimeoutSeconds(), TimeUnit.SECONDS, true); + return newClient(new RpcConnectionKey(addr, protocolClass, asyncMode), rpcParams); } + /** + * + * @param key RpcConnectionKey + * @param Rpc Protocol Class + * @return Rpc Client Class + * @throws NoSuchMethodException + * @throws ClassNotFoundException + * @throws ConnectException + */ public synchronized T newClient(RpcConnectionKey key, - int retries, - long timeout, - TimeUnit timeUnit, - boolean enablePing) + Properties connectionParameters) + throws NoSuchMethodException, ClassNotFoundException, ConnectException { - T client = makeClient(key, retries, timeout, timeUnit, enablePing); + T client = makeClient(key, connectionParameters); client.connect(); assert client.isConnected(); return client; @@ -165,12 +156,11 @@ public synchronized T newClient(RpcConnectionKey key public synchronized T newBlockingClient(InetSocketAddress addr, Class protocolClass, - int retries, - EventLoopGroup eventLoopGroup) + EventLoopGroup eventLoopGroup, + Properties rpcParams) throws NoSuchMethodException, ClassNotFoundException, ConnectException { - T client = makeClient(new RpcConnectionKey(addr, protocolClass, false), - retries, 0, TimeUnit.SECONDS, false, eventLoopGroup); + T client = makeClient(eventLoopGroup, new RpcConnectionKey(addr, protocolClass, false), rpcParams); client.connect(); assert client.isConnected(); return client; @@ -220,61 +210,10 @@ public static void cleanup(NettyClientBase... clients) { } } - public int getTimeoutSeconds() { - return timeoutSeconds; - } - - public void setTimeoutSeconds(int timeoutSeconds) { - this.timeoutSeconds = timeoutSeconds; - } - - public int getRetries() { - return retries; - } - - public void setRetries(int retries) { - this.retries = retries; - } - - static class RpcConnectionKey { - final InetSocketAddress addr; - final Class protocolClass; - final boolean asyncMode; - - final String description; - - public RpcConnectionKey(InetSocketAddress addr, - Class protocolClass, boolean asyncMode) { - this.addr = addr; - this.protocolClass = protocolClass; - this.asyncMode = asyncMode; - this.description = "[" + protocolClass + "] " + addr + "," + asyncMode; - } - - @Override - public String toString() { - return description; - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof RpcConnectionKey)) { - return false; - } - - return toString().equals(obj.toString()); - } - - @Override - public int hashCode() { - return description.hashCode(); - } - } - static class ClientCloseFutureListener implements GenericFutureListener { - private RpcClientManager.RpcConnectionKey key; + private RpcConnectionKey key; - public ClientCloseFutureListener(RpcClientManager.RpcConnectionKey key) { + public ClientCloseFutureListener(RpcConnectionKey key) { this.key = key; } diff --git a/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcConnectionKey.java b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcConnectionKey.java new file mode 100644 index 0000000000..2804010853 --- /dev/null +++ b/tajo-rpc/tajo-rpc-protobuf/src/main/java/org/apache/tajo/rpc/RpcConnectionKey.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.rpc; + +import java.net.InetSocketAddress; + +public class RpcConnectionKey { + final InetSocketAddress addr; + final Class protocolClass; + final boolean asyncMode; + + final String description; + + public RpcConnectionKey(InetSocketAddress addr, + Class protocolClass, boolean asyncMode) { + this.addr = addr; + this.protocolClass = protocolClass; + this.asyncMode = asyncMode; + this.description = "[" + protocolClass + "] " + addr + "," + asyncMode; + } + + @Override + public String toString() { + return description; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof RpcConnectionKey)) { + return false; + } + + return toString().equals(obj.toString()); + } + + @Override + public int hashCode() { + return description.hashCode(); + } +} diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java index 8a3f38584a..6427ffe6c0 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java @@ -42,6 +42,7 @@ import java.net.ConnectException; import java.net.InetSocketAddress; import java.net.ServerSocket; +import java.util.Properties; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -61,7 +62,7 @@ public class TestAsyncRpc { Interface stub; DummyProtocolAsyncImpl service; int retries; - RpcClientManager.RpcConnectionKey rpcConnectionKey; + RpcConnectionKey rpcConnectionKey; RpcClientManager manager = RpcClientManager.getInstance(); @Retention(RetentionPolicy.RUNTIME) @@ -129,10 +130,16 @@ public void setUpRpcServer() throws Exception { public void setUpRpcClient() throws Exception { retries = 1; - rpcConnectionKey = new RpcClientManager.RpcConnectionKey( - RpcUtils.getConnectAddress(server.getListenAddress()), - DummyProtocol.class, true); - client = manager.newClient(rpcConnectionKey, retries, 10, TimeUnit.SECONDS, true); + rpcConnectionKey = new RpcConnectionKey( + RpcUtils.getConnectAddress(server.getListenAddress()), DummyProtocol.class, true); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(10))); + connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); + + + client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); stub = client.getStub(); } @@ -347,10 +354,13 @@ public void run() { }); serverThread.start(); - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, true); - AsyncRpcClient client = manager.newClient(rpcConnectionKey, - retries, 0, TimeUnit.MILLISECONDS, false); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(address, DummyProtocol.class, true); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); + + AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Interface stub = client.getStub(); @@ -377,9 +387,13 @@ public void testClientRetryFailureOnStartup() throws Exception { .setMessage(MESSAGE).build(); CallFuture future = new CallFuture<>(); - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, true); - AsyncRpcClient client = new AsyncRpcClient(rpcConnectionKey, retries); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(address, DummyProtocol.class, true); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); + + AsyncRpcClient client = new AsyncRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); try { client.connect(); fail(); @@ -409,9 +423,13 @@ public void testUnresolvedAddress() throws Exception { boolean expected = false; AsyncRpcClient client = null; try { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, true); - client = new AsyncRpcClient(rpcConnectionKey, retries); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(address, DummyProtocol.class, true); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); + + client = new AsyncRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); client.connect(); fail(); } catch (ConnectException e) { @@ -429,10 +447,14 @@ public void testUnresolvedAddress() throws Exception { @SetupRpcConnection(setupRpcClient = false) public void testUnresolvedAddress2() throws Exception { String hostAndPort = RpcUtils.normalizeInetSocketAddress(server.getListenAddress()); - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey( + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey( RpcUtils.createUnresolved(hostAndPort), DummyProtocol.class, true); - AsyncRpcClient client = new AsyncRpcClient(rpcConnectionKey, retries); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); + + AsyncRpcClient client = new AsyncRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); client.connect(); try { assertTrue(client.isConnected()); @@ -453,9 +475,12 @@ public void testUnresolvedAddress2() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testStubRecovery() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); - AsyncRpcClient client = manager.newClient(rpcConnectionKey, 2, 0, TimeUnit.MILLISECONDS, false); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(2)); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); EchoMessage echoMessage = EchoMessage.newBuilder() .setMessage(MESSAGE).build(); @@ -484,10 +509,15 @@ public void testStubRecovery() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testIdleTimeout() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); - //500 millis idle timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, false); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + + // 500 millis idle timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + + AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Thread.sleep(600); //timeout @@ -504,11 +534,16 @@ public void testIdleTimeout() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testPingOnIdle() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); - //500 millis request timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, true); + // 500 millis idle timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); + + AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Thread.sleep(600); @@ -522,10 +557,15 @@ public void testPingOnIdle() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testIdleTimeoutWithActiveRequest() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); - //500 millis idle timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, false); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + + // 500 millis idle timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + + AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Interface stub = client.getStub(); @@ -547,11 +587,16 @@ public void testIdleTimeoutWithActiveRequest() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testRequestTimeoutOnBusy() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + + // 500 millis idle timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(retries)); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); - //500 millis request timeout - AsyncRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, true); + AsyncRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Interface stub = client.getStub(); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java index 0fae7ee093..0687d0ba35 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java @@ -39,6 +39,7 @@ import java.net.ConnectException; import java.net.InetSocketAddress; import java.net.ServerSocket; +import java.util.Properties; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -119,11 +120,16 @@ public void setUpRpcServer() throws Exception { public void setUpRpcClient() throws Exception { retries = 1; - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey( + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, "1"); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(TimeUnit.SECONDS.toMillis(10))); + connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); + + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey( RpcUtils.getConnectAddress(server.getListenAddress()), DummyProtocol.class, false); - client = manager.newClient(rpcConnectionKey, retries, 10, TimeUnit.SECONDS, true); + client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); stub = client.getStub(); } @@ -317,11 +323,13 @@ public void run() { }); serverThread.start(); - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, false); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(address, DummyProtocol.class, false); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); - BlockingRpcClient client = manager.newClient(rpcConnectionKey, - retries, 0, TimeUnit.MILLISECONDS, false); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); BlockingInterface stub = client.getStub(); @@ -342,9 +350,14 @@ public void testClientRetryFailureOnStartup() throws Exception { EchoMessage message = EchoMessage.newBuilder() .setMessage(MESSAGE).build(); - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, false); - BlockingRpcClient client = new BlockingRpcClient(rpcConnectionKey, retries); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(address, DummyProtocol.class, false); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); + + BlockingRpcClient client = new BlockingRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, + connParams); try { client.connect(); @@ -370,9 +383,13 @@ public void testUnresolvedAddress() throws Exception { boolean expected = false; BlockingRpcClient client = null; try { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(address, DummyProtocol.class, true); - client = new BlockingRpcClient(rpcConnectionKey, retries); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(address, DummyProtocol.class, true); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); + + client = new BlockingRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); client.connect(); fail(); } catch (ConnectException e) { @@ -388,11 +405,17 @@ public void testUnresolvedAddress() throws Exception { @Test(timeout = 120000) @SetupRpcConnection(setupRpcClient = false) public void testUnresolvedAddress2() throws Exception { + String hostAndPort = RpcUtils.normalizeInetSocketAddress(server.getListenAddress()); - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey( + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey( RpcUtils.createUnresolved(hostAndPort), DummyProtocol.class, false); - BlockingRpcClient client = new BlockingRpcClient(rpcConnectionKey, retries); + + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); + + BlockingRpcClient client = + new BlockingRpcClient(NettyUtils.getDefaultEventLoopGroup(), rpcConnectionKey, connParams); client.connect(); assertTrue(client.isConnected()); @@ -410,9 +433,11 @@ public void testUnresolvedAddress2() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testStubRecovery() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); - BlockingRpcClient client = manager.newClient(rpcConnectionKey, 1, 0, TimeUnit.MILLISECONDS, false); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, String.valueOf(1)); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); EchoMessage echoMessage = EchoMessage.newBuilder() .setMessage(MESSAGE).build(); @@ -440,10 +465,15 @@ public void testStubRecovery() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testIdleTimeout() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); - //500 millis idle timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, false); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + + // 500 millis socket timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + + BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Thread.sleep(600); //timeout @@ -460,11 +490,16 @@ public void testIdleTimeout() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testPingOnIdle() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + + // 500 millis socket timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); - //500 millis request timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, true); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); Thread.sleep(600); @@ -478,10 +513,15 @@ public void testPingOnIdle() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testIdleTimeoutWithActiveRequest() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); - //500 millis idle timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, false); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + + // 500 millis socket timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + + BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); BlockingInterface stub = client.getStub(); @@ -500,11 +540,16 @@ public void testIdleTimeoutWithActiveRequest() throws Exception { @Test(timeout = 60000) @SetupRpcConnection(setupRpcClient = false) public void testRequestTimeoutOnBusy() throws Exception { - RpcClientManager.RpcConnectionKey rpcConnectionKey = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + RpcConnectionKey rpcConnectionKey = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, false); + + // 500 millis socket timeout + Properties connParams = new Properties(); + connParams.setProperty(RpcConstants.CLIENT_RETRY_NUM, retries + ""); + connParams.setProperty(RpcConstants.CLIENT_SOCKET_TIMEOUT, String.valueOf(500)); + connParams.setProperty(RpcConstants.CLIENT_HANG_DETECTION, "true"); - //500 millis request timeout - BlockingRpcClient client = manager.newClient(rpcConnectionKey, retries, 500, TimeUnit.MILLISECONDS, true); + BlockingRpcClient client = manager.newClient(rpcConnectionKey, connParams); assertTrue(client.isConnected()); BlockingInterface stub = client.getStub(); diff --git a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java index 1053de6c70..160c6a3074 100644 --- a/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java +++ b/tajo-rpc/tajo-rpc-protobuf/src/test/java/org/apache/tajo/rpc/TestRpcClientManager.java @@ -25,10 +25,10 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; +import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import static org.junit.Assert.*; @@ -54,7 +54,7 @@ public void testRaceCondition() throws Exception { public void run() { NettyClientBase client = null; try { - client = manager.getClient(address, DummyProtocol.class, false); + client = manager.getClient(address, DummyProtocol.class, false, new Properties()); } catch (Throwable e) { fail(e.getMessage()); } @@ -68,7 +68,7 @@ public void run() { future.get(); } - NettyClientBase clientBase = manager.getClient(address, DummyProtocol.class, false); + NettyClientBase clientBase = manager.getClient(address, DummyProtocol.class, false, new Properties()); RpcClientManager.cleanup(clientBase); } finally { server.shutdown(); @@ -87,11 +87,11 @@ public void testClientCloseEvent() throws Exception { try { - NettyClientBase client = manager.getClient(server.getListenAddress(), DummyProtocol.class, true); + NettyClientBase client = manager.getClient(server.getListenAddress(), DummyProtocol.class, true, new Properties()); assertTrue(client.isConnected()); assertTrue(client.getChannel().isWritable()); - RpcClientManager.RpcConnectionKey key = client.getKey(); + RpcConnectionKey key = client.getKey(); assertTrue(RpcClientManager.contains(key)); client.close(); @@ -113,10 +113,10 @@ public void testClientCloseEventWithReconnect() throws Exception { try { - NettyClientBase client = manager.getClient(server.getListenAddress(), DummyProtocol.class, true); + NettyClientBase client = manager.getClient(server.getListenAddress(), DummyProtocol.class, true, new Properties()); assertTrue(client.isConnected()); - RpcClientManager.RpcConnectionKey key = client.getKey(); + RpcConnectionKey key = client.getKey(); assertTrue(RpcClientManager.contains(key)); client.close(); @@ -144,17 +144,17 @@ public void testUnManagedClient() throws Exception { NettyServerBase server = new AsyncRpcServer(DummyProtocol.class, service, new InetSocketAddress("127.0.0.1", 0), 3); server.start(); - RpcClientManager.RpcConnectionKey key = - new RpcClientManager.RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); + RpcConnectionKey key = + new RpcConnectionKey(server.getListenAddress(), DummyProtocol.class, true); RpcClientManager.close(); RpcClientManager manager = RpcClientManager.getInstance(); try { - NettyClientBase client1 = manager.newClient(key, 0, 0, TimeUnit.SECONDS, false); + NettyClientBase client1 = manager.newClient(key, new Properties()); assertTrue(client1.isConnected()); assertFalse(RpcClientManager.contains(key)); - NettyClientBase client2 = manager.newClient(key, 0, 0, TimeUnit.SECONDS, false); + NettyClientBase client2 = manager.newClient(key, new Properties()); assertTrue(client2.isConnected()); assertFalse(RpcClientManager.contains(key)); From aa699c70943f7563e77f5828683cd0399998a0f9 Mon Sep 17 00:00:00 2001 From: Dongkyu Hwangbo Date: Tue, 22 Sep 2015 21:24:06 -0700 Subject: [PATCH 04/90] TAJO-1844: Eliminate explicit diamond expressions. Signed-off-by: Hyunsik Choi --- CHANGES | 3 + .../tajo/catalog/AbstractCatalogClient.java | 2 +- .../org/apache/tajo/catalog/CatalogUtil.java | 6 +- .../org/apache/tajo/catalog/FunctionDesc.java | 2 +- .../java/org/apache/tajo/catalog/Schema.java | 6 +- .../org/apache/tajo/catalog/SchemaUtil.java | 2 +- .../tajo/catalog/statistics/TableStats.java | 4 +- .../catalog/statistics/TestTableStat.java | 2 +- .../tajo/catalog/store/HiveCatalogStore.java | 6 +- .../store/HiveCatalogStoreClientPool.java | 2 +- .../catalog/store/TestHiveCatalogStore.java | 2 +- .../apache/tajo/catalog/CatalogServer.java | 5 +- .../tajo/catalog/LinkedMetadataManager.java | 4 +- .../InfoSchemaMetadataDictionary.java | 4 +- .../tajo/catalog/store/AbstractDBStore.java | 22 +++--- .../store/XMLCatalogSchemaManager.java | 22 +++--- .../tajo/catalog/store/object/BaseSchema.java | 2 +- .../catalog/store/object/SchemaPatch.java | 2 +- .../catalog/store/object/StoreObject.java | 6 +- .../org/apache/tajo/catalog/TestCatalog.java | 6 +- .../store/TestXMLCatalogSchemaManager.java | 4 +- .../org/apache/tajo/cli/tools/TajoAdmin.java | 10 +-- .../org/apache/tajo/cli/tools/TajoDump.java | 4 +- .../apache/tajo/cli/tsql/SimpleParser.java | 6 +- .../org/apache/tajo/cli/tsql/TajoCli.java | 2 +- .../tsql/commands/DescFunctionCommand.java | 4 +- .../tajo/cli/tsql/commands/SetCommand.java | 2 +- .../tajo/client/DummyServiceTracker.java | 2 +- .../apache/tajo/client/SessionConnection.java | 2 +- .../org/apache/tajo/jdbc/MetaDataTuple.java | 4 +- .../org/apache/tajo/QueryTestCaseBase.java | 8 +-- .../org/apache/tajo/TajoTestingCluster.java | 2 +- .../java/org/apache/tajo/QueryIdFactory.java | 2 +- .../apache/tajo/exception/ErrorMessages.java | 2 +- .../tajo/rule/base/BaseRuleProvider.java | 2 +- .../tajo/util/datetime/DateTimeConstants.java | 2 +- .../tajo/util/datetime/DateTimeFormat.java | 4 +- .../tajo/util/graph/DirectedGraphCursor.java | 2 +- .../tajo/util/graph/SimpleDirectedGraph.java | 8 +-- .../tsql/TestDefaultCliOutputFormatter.java | 2 +- .../apache/tajo/client/TestTajoClient.java | 4 +- .../tajo/engine/eval/TestEvalTreeUtil.java | 2 +- .../tajo/engine/planner/TestLogicalPlan.java | 4 +- .../engine/planner/TestLogicalPlanner.java | 2 +- .../tajo/engine/planner/TestPlannerUtil.java | 2 +- .../planner/physical/TestHashPartitioner.java | 2 +- .../planner/physical/TestPhysicalPlanner.java | 4 +- .../planner/physical/TestTupleSorter.java | 2 +- .../tajo/engine/query/TestGroupByQuery.java | 6 +- .../tajo/engine/query/TestHBaseTable.java | 14 ++-- .../tajo/engine/query/TestIndexScan.java | 2 +- .../tajo/engine/query/TestJoinQuery.java | 2 +- .../engine/query/TestJsonWithTimezone.java | 2 +- .../tajo/engine/query/TestSelectQuery.java | 2 +- .../engine/query/TestTablePartitions.java | 2 +- .../tajo/engine/util/TestTableCache.java | 2 +- .../apache/tajo/master/TestRepartitioner.java | 26 +++---- .../master/scheduler/TestSimpleScheduler.java | 4 +- .../querymaster/TestIntermediateEntry.java | 2 +- .../org/apache/tajo/util/TestJSPUtil.java | 4 +- .../util/history/TestHistoryWriterReader.java | 4 +- .../tajo/util/metrics/TestMetricsFilter.java | 2 +- .../tajo/util/metrics/TestSystemMetrics.java | 2 +- .../tajo/worker/MockNodeResourceManager.java | 2 +- .../tajo/worker/TestNodeResourceManager.java | 10 +-- .../apache/tajo/worker/TestTaskExecutor.java | 4 +- .../apache/tajo/worker/TestTaskManager.java | 2 +- .../ws/rs/resources/TestSessionsResource.java | 6 +- .../apache/tajo/benchmark/BenchmarkSet.java | 6 +- .../engine/codegen/ExecutorPreCompiler.java | 2 +- .../engine/codegen/TajoGeneratorAdapter.java | 2 +- .../tajo/engine/function/FunctionLoader.java | 2 +- .../tajo/engine/function/builtin/Lead.java | 2 +- .../engine/planner/PhysicalPlannerImpl.java | 2 +- .../engine/planner/global/ExecutionBlock.java | 6 +- .../planner/global/ExecutionBlockCursor.java | 12 ++-- .../engine/planner/global/GlobalPlanner.java | 2 +- .../engine/planner/global/MasterPlan.java | 6 +- .../global/ParallelExecutionQueue.java | 10 +-- .../builder/DistinctGroupbyBuilder.java | 46 ++++++------- .../rewriter/GlobalPlanRewriteEngine.java | 2 +- .../planner/physical/BSTIndexScanExec.java | 2 +- .../planner/physical/CommonHashJoinExec.java | 2 +- .../DistinctGroupbyFirstAggregationExec.java | 6 +- .../DistinctGroupbyHashAggregationExec.java | 8 +-- .../DistinctGroupbySecondAggregationExec.java | 8 +-- .../DistinctGroupbyThirdAggregationExec.java | 6 +- .../planner/physical/HashAggregateExec.java | 2 +- .../HashBasedColPartitionStoreExec.java | 4 +- .../physical/HashFullOuterJoinExec.java | 2 +- .../physical/HashShuffleFileWriteExec.java | 4 +- .../planner/physical/PhysicalPlanUtil.java | 4 +- .../engine/planner/physical/SeqScanExec.java | 2 +- .../tajo/engine/query/QueryContext.java | 2 +- .../tajo/engine/query/TaskRequestImpl.java | 4 +- .../apache/tajo/ha/HdfsServiceTracker.java | 2 +- .../apache/tajo/master/QueryInProgress.java | 6 +- .../org/apache/tajo/master/QueryManager.java | 4 +- .../tajo/master/TajoMasterClientService.java | 6 +- .../apache/tajo/master/exec/DDLExecutor.java | 2 +- .../exec/ExplainPlanPreprocessorForTest.java | 2 +- .../NonForwardQueryResultSystemScanner.java | 24 +++---- .../tajo/master/exec/QueryExecutor.java | 2 +- .../prehook/DistributedQueryHookManager.java | 2 +- .../tajo/master/rule/MasterRuleProvider.java | 2 +- .../master/scheduler/SimpleScheduler.java | 6 +- .../metrics/ClusterResourceMetricSet.java | 2 +- .../apache/tajo/parser/sql/SQLAnalyzer.java | 16 ++--- .../querymaster/DefaultTaskScheduler.java | 18 ++--- .../org/apache/tajo/querymaster/Query.java | 8 +-- .../apache/tajo/querymaster/QueryMaster.java | 10 +-- .../QueryMasterManagerService.java | 2 +- .../tajo/querymaster/QueryMasterTask.java | 4 +- .../tajo/querymaster/Repartitioner.java | 69 +++++++++---------- .../org/apache/tajo/querymaster/Stage.java | 6 +- .../org/apache/tajo/querymaster/Task.java | 26 +++---- .../apache/tajo/querymaster/TaskAttempt.java | 4 +- .../java/org/apache/tajo/session/Session.java | 4 +- .../apache/tajo/session/SessionManager.java | 2 +- .../java/org/apache/tajo/util/JSPUtil.java | 10 +-- .../tajo/util/history/HistoryReader.java | 4 +- .../tajo/util/history/HistoryWriter.java | 8 +-- .../tajo/util/history/QueryHistory.java | 4 +- .../tajo/util/history/StageHistory.java | 2 +- .../tajo/util/metrics/LogEventGaugeSet.java | 2 +- .../tajo/util/metrics/MetricsFilterList.java | 2 +- .../util/metrics/RegexpMetricsFilter.java | 2 +- .../tajo/util/metrics/TajoSystemMetrics.java | 8 +-- .../metrics/reporter/TajoMetricsReporter.java | 6 +- .../TajoMetricsScheduledReporter.java | 2 +- .../org/apache/tajo/webapp/HttpServer.java | 6 +- .../tajo/webapp/QueryExecutorServlet.java | 14 ++-- .../tajo/worker/ExecutionBlockContext.java | 8 +-- .../worker/ExecutionBlockSharedResource.java | 2 +- .../org/apache/tajo/worker/FetchImpl.java | 4 +- .../apache/tajo/worker/NodeStatusUpdater.java | 2 +- .../org/apache/tajo/worker/TajoWorker.java | 2 +- .../tajo/worker/TaskAttemptContext.java | 6 +- .../org/apache/tajo/worker/TaskExecutor.java | 2 +- .../java/org/apache/tajo/worker/TaskImpl.java | 4 +- .../org/apache/tajo/worker/TaskManager.java | 2 +- .../tajo/worker/rule/WorkerRuleProvider.java | 2 +- .../apache/tajo/ws/rs/ClientApplication.java | 4 +- .../ws/rs/JerseyResourceDelegateContext.java | 2 +- .../rs/JerseyResourceDelegateContextKey.java | 4 +- .../tajo/ws/rs/resources/ClusterResource.java | 4 +- .../ws/rs/resources/DatabasesResource.java | 2 +- .../ws/rs/resources/FunctionsResource.java | 4 +- .../tajo/ws/rs/resources/QueryResource.java | 8 +-- .../ws/rs/resources/SessionsResource.java | 4 +- .../tajo/ws/rs/resources/TablesResource.java | 2 +- .../main/resources/webapps/admin/cluster.jsp | 12 ++-- .../resources/webapps/admin/functions.jsp | 2 +- .../main/resources/webapps/admin/query.jsp | 6 +- .../resources/webapps/admin/querytasks.jsp | 4 +- .../resources/webapps/worker/queryplan.jsp | 4 +- .../resources/webapps/worker/querytasks.jsp | 4 +- .../tajo/jdbc/TajoDatabaseMetaData.java | 12 ++-- .../tajo/jdbc/TajoPreparedStatement.java | 2 +- .../tajo/jdbc/util/QueryStringDecoder.java | 4 +- .../tajo/jdbc/TestTajoDatabaseMetaData.java | 6 +- .../org/apache/tajo/jdbc/TestTajoJdbc.java | 4 +- .../tajo/maven/plugin/protoc/ProtocMojo.java | 8 +-- .../apache/tajo/maven/plugin/util/Exec.java | 2 +- .../plugin/versioninfo/VersionInfoMojo.java | 2 +- .../org/apache/tajo/plan/ExprAnnotator.java | 4 +- .../org/apache/tajo/plan/ExprNormalizer.java | 8 +-- .../apache/tajo/plan/LogicalOptimizer.java | 8 +-- .../org/apache/tajo/plan/LogicalPlan.java | 12 ++-- .../org/apache/tajo/plan/LogicalPlanner.java | 16 ++--- .../java/org/apache/tajo/plan/PlanString.java | 4 +- .../apache/tajo/plan/expr/AlgebraicUtil.java | 6 +- .../apache/tajo/plan/expr/CaseWhenEval.java | 2 +- .../apache/tajo/plan/expr/EvalTreeUtil.java | 2 +- .../exprrewrite/rules/ConstantFolding.java | 2 +- .../tajo/plan/joinorder/JoinGraphContext.java | 4 +- .../plan/logical/DistinctGroupbyNode.java | 4 +- .../apache/tajo/plan/logical/JoinSpec.java | 2 +- .../plan/rewrite/BaseSchemaBuildPhase.java | 2 +- .../rewrite/rules/ProjectionPushDownRule.java | 14 ++-- .../apache/tajo/plan/util/PlannerUtil.java | 4 +- .../visitor/ExplainLogicalPlanVisitor.java | 4 +- .../pullserver/HttpDataServerHandler.java | 4 +- .../pullserver/TajoPullServerService.java | 6 +- .../retriever/AdvancedDataRetriever.java | 2 +- .../java/org/apache/tajo/rpc/NettyUtils.java | 2 +- .../apache/tajo/storage/DiskDeviceInfo.java | 2 +- .../org/apache/tajo/storage/DiskUtil.java | 8 +-- .../org/apache/tajo/storage/MergeScanner.java | 2 +- .../tajo/storage/OldStorageManager.java | 4 +- .../tajo/storage/compress/CodecPool.java | 4 +- .../tajo/storage/TestTupleComparator.java | 4 +- .../storage/hbase/AbstractHBaseAppender.java | 4 +- .../tajo/storage/hbase/ColumnMapping.java | 2 +- .../tajo/storage/hbase/HBaseTablespace.java | 28 ++++---- .../tajo/storage/hbase/HFileAppender.java | 2 +- .../storage/hbase/SortedInsertRewriter.java | 2 +- .../apache/tajo/storage/FileTablespace.java | 14 ++-- .../tajo/storage/HashShuffleAppender.java | 8 +-- .../storage/HashShuffleAppenderManager.java | 8 +-- .../tajo/storage/avro/AvroAppender.java | 4 +- .../apache/tajo/storage/avro/AvroScanner.java | 4 +- .../tajo/storage/fragment/FileFragment.java | 2 +- .../tajo/storage/index/bst/BSTIndex.java | 4 +- .../apache/tajo/storage/orc/ORCScanner.java | 2 +- .../TajoStructObjectInspector.java | 2 +- .../storage/parquet/TajoSchemaConverter.java | 4 +- .../storage/parquet/TajoWriteSupport.java | 2 +- .../storage/rcfile/ColumnProjectionUtils.java | 4 +- .../tajo/storage/text/DelimitedTextFile.java | 2 +- .../thirdparty/orc/FileOrcDataSource.java | 2 +- .../thirdparty/orc/HdfsOrcDataSource.java | 2 +- .../storage/thirdparty/orc/MemoryManager.java | 2 +- .../tajo/storage/thirdparty/orc/OrcUtils.java | 2 +- .../storage/thirdparty/orc/WriterImpl.java | 12 ++-- .../thirdparty/parquet/CodecFactory.java | 6 +- .../parquet/ColumnChunkPageWriteStore.java | 6 +- .../thirdparty/parquet/ParquetFileWriter.java | 12 ++-- .../thirdparty/parquet/ParquetReader.java | 4 +- .../thirdparty/parquet/ParquetWriter.java | 24 +++---- .../tajo/storage/parquet/TestReadWrite.java | 2 +- .../storage/parquet/TestSchemaConverter.java | 4 +- .../objectweb/asm/commons/AdviceAdapter.java | 8 +-- .../asm/commons/AnalyzerAdapter.java | 14 ++-- .../asm/commons/GeneratorAdapter.java | 2 +- .../asm/commons/JSRInlinerAdapter.java | 10 +-- .../org/objectweb/asm/commons/Method.java | 2 +- .../asm/commons/SerialVersionUIDAdder.java | 6 +- .../objectweb/asm/optimizer/JarOptimizer.java | 4 +- .../objectweb/asm/optimizer/NameMapping.java | 2 +- .../org/objectweb/asm/optimizer/Shrinker.java | 6 +- .../objectweb/asm/tree/AnnotationNode.java | 10 +-- .../org/objectweb/asm/tree/ClassNode.java | 14 ++-- .../org/objectweb/asm/tree/FieldNode.java | 6 +- .../org/objectweb/asm/tree/FrameNode.java | 4 +- .../asm/tree/LookupSwitchInsnNode.java | 4 +- .../org/objectweb/asm/tree/MethodNode.java | 16 ++--- .../asm/tree/TableSwitchInsnNode.java | 2 +- .../objectweb/asm/tree/analysis/Analyzer.java | 10 +-- .../objectweb/asm/tree/analysis/Frame.java | 6 +- .../objectweb/asm/tree/analysis/SmallSet.java | 8 +-- .../asm/tree/analysis/SourceInterpreter.java | 2 +- .../asm/tree/analysis/SourceValue.java | 2 +- .../asm/tree/analysis/Subroutine.java | 4 +- .../tajo/org/objectweb/asm/util/ASMifier.java | 4 +- .../objectweb/asm/util/CheckClassAdapter.java | 6 +- .../asm/util/CheckMethodAdapter.java | 6 +- .../tajo/org/objectweb/asm/util/Printer.java | 2 +- .../org/objectweb/asm/util/Textifier.java | 2 +- .../objectweb/asm/xml/ASMContentHandler.java | 30 ++++---- .../org/objectweb/asm/xml/SAXCodeAdapter.java | 2 +- 251 files changed, 704 insertions(+), 703 deletions(-) diff --git a/CHANGES b/CHANGES index cfad5fe4aa..5e929ab644 100644 --- a/CHANGES +++ b/CHANGES @@ -669,6 +669,9 @@ Release 0.11.0 - unreleased SUB TASKS + TAJO-1844: Eliminate explicit diamond expressions. + (Contributed by Dongkyu Hwangbo, committed by hyunsik) + TAJO-1465: Add ORCFileAppender to write into ORCFile table. (Contributed by Joyngyoung Park, committed by hyunsik) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index 1dc7a71af1..7fb3bed9ad 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -512,7 +512,7 @@ public final Collection getAllTableNames(final String databaseName) { @Override public final Collection getFunctions() { - List list = new ArrayList(); + List list = new ArrayList<>(); try { final BlockingInterface stub = getStub(); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 4e52de1adb..321b8bac60 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -234,7 +234,7 @@ public static String buildFQName(String... identifiers) { public static Pair separateQualifierAndName(String name) { Preconditions.checkArgument(isFQTableName(name), "Must be a qualified name."); - return new Pair(extractQualifier(name), extractSimpleName(name)); + return new Pair<>(extractQualifier(name), extractSimpleName(name)); } /** @@ -741,7 +741,7 @@ public static void closeQuietly(Statement stmt, ResultSet res) { } } - public static final Set RESERVED_KEYWORDS_SET = new HashSet(); + public static final Set RESERVED_KEYWORDS_SET = new HashSet<>(); static final String [] RESERVED_KEYWORDS = { "AS", "ALL", "AND", "ANY", "ASYMMETRIC", "ASC", @@ -869,7 +869,7 @@ public static Pair, String> getPartitionKeyNamePair(Stri partitionKeyList.add(builder.build()); } - pair = new Pair, String>(partitionKeyList, sb.toString()); + pair = new Pair<>(partitionKeyList, sb.toString()); return pair; } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java index a2d4b75db9..5fa0c15f7a 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FunctionDesc.java @@ -50,7 +50,7 @@ public FunctionDesc(String signature, Class clazz, FunctionType funcType, DataType retType, @NotNull DataType [] params) { this.signature = new FunctionSignature(funcType, signature.toLowerCase(), retType, params); this.invocation = new FunctionInvocation(); - this.invocation.setLegacy(new ClassBaseInvocationDesc(clazz)); + this.invocation.setLegacy(new ClassBaseInvocationDesc<>(clazz)); this.supplement = new FunctionSupplement(); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java index abd3bcaecb..ae7d27446e 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Schema.java @@ -122,9 +122,9 @@ public Schema(Iterable columns) { } private void init() { - this.fields = new ArrayList(); - this.fieldsByQualifiedName = new HashMap(); - this.fieldsByName = new HashMap>(); + this.fields = new ArrayList<>(); + this.fieldsByQualifiedName = new HashMap<>(); + this.fieldsByName = new HashMap<>(); } /** diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java index 4a8e419d2b..6af9e067e1 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java @@ -133,7 +133,7 @@ public static T clone(Schema schema) { public static ImmutableMap buildTypeMap(Iterable schema, String [] targetPaths) { - HashMap builder = new HashMap(); + HashMap builder = new HashMap<>(); for (Column column : schema) { // Keep types which only belong to projected paths diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java index dd358ae1de..9ff1fba973 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java @@ -145,7 +145,7 @@ public List getColumnStats() { } public void setColumnStats(List columnStatses) { - this.columnStatses = new ArrayList(columnStatses); + this.columnStatses = new ArrayList<>(columnStatses); } public void addColumnStat(ColumnStats columnStats) { @@ -183,7 +183,7 @@ public Object clone() throws CloneNotSupportedException { stat.avgRows = avgRows != null ? avgRows : null; stat.readBytes = readBytes != null ? readBytes : null; - stat.columnStatses = new ArrayList(this.columnStatses); + stat.columnStatses = new ArrayList<>(this.columnStatses); return stat; } diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestTableStat.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestTableStat.java index b649dd7353..0356416db2 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestTableStat.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/statistics/TestTableStat.java @@ -89,7 +89,7 @@ public void tableStatEquals(TableStats s1, TableStats s2) { public void testGetProtoThreadSafe() throws Exception { final TableStats tableStats = new TableStats(); - List columnStatsList = new ArrayList(); + List columnStatsList = new ArrayList<>(); for (int i = 0; i < 3; i++) { Column column = new Column("col_" + (i + 1), Type.TEXT); ColumnStats columnStats = new ColumnStats(column); diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index e2229baca3..e4d53b25cb 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -426,7 +426,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) org.apache.hadoop.hive.metastore.api.Table table = new org.apache.hadoop.hive.metastore.api.Table(); table.setDbName(databaseName); table.setTableName(tableName); - table.setParameters(new HashMap(tableDesc.getMeta().getOptions().getAllKeyValus())); + table.setParameters(new HashMap<>(tableDesc.getMeta().getOptions().getAllKeyValus())); // TODO: set owner //table.setOwner(); @@ -454,7 +454,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) // set column information List columns = tableDesc.getSchema().getRootColumns(); - ArrayList cols = new ArrayList(columns.size()); + ArrayList cols = new ArrayList<>(columns.size()); for (Column eachField : columns) { cols.add(new FieldSchema(eachField.getSimpleName(), @@ -464,7 +464,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) // set partition keys if (tableDesc.hasPartition() && tableDesc.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) { - List partitionKeys = new ArrayList(); + List partitionKeys = new ArrayList<>(); for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) { partitionKeys.add(new FieldSchema(eachPartitionKey.getSimpleName(), HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()), "")); diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStoreClientPool.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStoreClientPool.java index 9053c5661b..bfde8cf985 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStoreClientPool.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStoreClientPool.java @@ -34,7 +34,7 @@ public class HiveCatalogStoreClientPool { private static final Logger LOG = Logger.getLogger(HiveCatalogStoreClientPool.class); private final ConcurrentLinkedQueue clientPool = - new ConcurrentLinkedQueue(); + new ConcurrentLinkedQueue<>(); private AtomicBoolean poolClosed = new AtomicBoolean(false); private HiveConf hiveConf; diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index b3af179421..3abcf8b06a 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -286,7 +286,7 @@ private void testAddPartition(URI uri, String tableName, String partitionName) t PartitionDesc partitionDesc = new PartitionDesc(); partitionDesc.setPartitionName(partitionName); - List partitionKeyList = new ArrayList(); + List partitionKeyList = new ArrayList<>(); String[] partitionNames = partitionName.split("/"); for(int i = 0; i < partitionNames.length; i++) { String[] eachPartitionName = partitionNames[i].split("="); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index a60dddab34..e02705981d 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -82,8 +82,7 @@ public class CatalogServer extends AbstractService { private final Lock wlock = lock.writeLock(); private CatalogStore store; - private Map> functions = new ConcurrentHashMap>(); + private Map> functions = new ConcurrentHashMap<>(); protected LinkedMetadataManager linkedMetadataManager; protected final InfoSchemaMetadataDictionary metaDictionary = new InfoSchemaMetadataDictionary(); @@ -100,7 +99,7 @@ public CatalogServer() throws IOException { super(CatalogServer.class.getName()); this.handler = new CatalogProtocolHandler(); this.linkedMetadataManager = new LinkedMetadataManager(Collections.EMPTY_LIST); - this.builtingFuncs = new ArrayList(); + this.builtingFuncs = new ArrayList<>(); } public CatalogServer(Collection metadataProviders, Collection sqlFuncs) diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java index a4c718ea26..e6fd2850f6 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/LinkedMetadataManager.java @@ -88,7 +88,7 @@ public boolean apply(@Nullable MetadataProvider input) { return Optional.absent(); } else { MetadataProvider found = filtered.iterator().next(); - return Optional.of(new Pair(found.getTablespaceName(), found.getTablespaceUri())); + return Optional.of(new Pair<>(found.getTablespaceName(), found.getTablespaceUri())); } } @@ -101,7 +101,7 @@ public boolean apply(@Nullable MetadataProvider input) { public Collection> getTablespaces() { ImmutableList.Builder> builder = ImmutableList.builder(); for (MetadataProvider p : providerMap.values()) { - builder.add(new Pair(p.getDatabaseName(), p.getTablespaceUri())); + builder.add(new Pair<>(p.getDatabaseName(), p.getTablespaceUri())); } return builder.build(); } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java index 17f84d85da..2de98075d1 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/InfoSchemaMetadataDictionary.java @@ -44,8 +44,8 @@ private enum DEFINED_TABLES { MAX_TABLE } - private List schemaInfoTableDescriptors = new ArrayList( - Collections.nCopies(DEFINED_TABLES.MAX_TABLE.ordinal(), (TableDescriptor)null)); + private List schemaInfoTableDescriptors = new ArrayList<>( + Collections.nCopies(DEFINED_TABLES.MAX_TABLE.ordinal(), (TableDescriptor) null)); public InfoSchemaMetadataDictionary() { createSystemTableDescriptors(); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index d46ab3c085..5494d0a0a6 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -428,7 +428,7 @@ private Collection getAllTablespaceNamesInternal(@Nullable String whereC PreparedStatement pstmt = null; ResultSet resultSet = null; - List tablespaceNames = new ArrayList(); + List tablespaceNames = new ArrayList<>(); try { String sql = "SELECT SPACE_NAME FROM " + TB_SPACES; @@ -679,7 +679,7 @@ private Collection getAllDatabaseNamesInternal(@Nullable String whereCon PreparedStatement pstmt = null; ResultSet resultSet = null; - List databaseNames = new ArrayList(); + List databaseNames = new ArrayList<>(); try { String sql = "SELECT DB_NAME FROM " + TB_DATABASES; @@ -709,7 +709,7 @@ public List getAllDatabases() { Statement stmt = null; ResultSet resultSet = null; - List databases = new ArrayList(); + List databases = new ArrayList<>(); try { String sql = "SELECT DB_ID, DB_NAME, SPACE_ID FROM " + TB_DATABASES; @@ -1814,7 +1814,7 @@ public List getAllTableNames(String databaseName) throws UndefinedDataba PreparedStatement pstmt = null; ResultSet res = null; - List tables = new ArrayList(); + List tables = new ArrayList<>(); try { @@ -1847,7 +1847,7 @@ public List getAllTables() { Statement stmt = null; ResultSet resultSet = null; - List tables = new ArrayList(); + List tables = new ArrayList<>(); try { String sql = "SELECT t.TID, t.DB_ID, t." + COL_TABLES_NAME + ", t.TABLE_TYPE, t.PATH, t.STORE_TYPE, " + @@ -1896,7 +1896,7 @@ public List getAllTableProperties() { Statement stmt = null; ResultSet resultSet = null; - List options = new ArrayList(); + List options = new ArrayList<>(); try { String sql = "SELECT tid, key_, value_ FROM " + TB_OPTIONS; @@ -1931,7 +1931,7 @@ public List getAllTableStats() { Statement stmt = null; ResultSet resultSet = null; - List stats = new ArrayList(); + List stats = new ArrayList<>(); try { String sql = "SELECT tid, num_rows, num_bytes FROM " + TB_STATISTICS; @@ -1963,7 +1963,7 @@ public List getAllColumns() { Statement stmt = null; ResultSet resultSet = null; - List columns = new ArrayList(); + List columns = new ArrayList<>(); try { String sql = @@ -2198,7 +2198,7 @@ public List getPartitions(String databaseName, String tableN ResultSet res = null; PreparedStatement pstmt = null; PartitionDescProto.Builder builder = null; - List partitions = new ArrayList(); + List partitions = new ArrayList<>(); final int databaseId = getDatabaseId(databaseName); final int tableId = getTableId(databaseId, databaseName, tableName); @@ -2238,7 +2238,7 @@ public List getAllPartitions() { Statement stmt = null; ResultSet resultSet = null; - List partitions = new ArrayList(); + List partitions = new ArrayList<>(); try { String sql = "SELECT " + COL_PARTITIONS_PK + ", " + COL_TABLES_PK + ", PARTITION_NAME, " + @@ -2731,7 +2731,7 @@ public List getAllIndexNamesByTable(final String databaseName, final Str ResultSet res = null; PreparedStatement pstmt = null; - final List indexNames = new ArrayList(); + final List indexNames = new ArrayList<>(); try { final int databaseId = getDatabaseId(databaseName); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java index 0ecd90a377..65a8c5b5b1 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/XMLCatalogSchemaManager.java @@ -93,7 +93,7 @@ public void dropBaseSchema(Connection conn) { throw new TajoInternalError("Schema files are not loaded yet."); } - List failedObjects = new ArrayList(); + List failedObjects = new ArrayList<>(); Statement stmt = null; try { @@ -304,7 +304,7 @@ public void upgradeBaseSchema(Connection conn, int currentVersion) { throw new TajoInternalError("Database schema files are not loaded."); } - final List candidatePatches = new ArrayList(); + final List candidatePatches = new ArrayList<>(); Statement stmt; for (SchemaPatch patch: this.catalogStore.getPatches()) { @@ -392,7 +392,7 @@ protected String[] listFileResources(URL dirURL, String schemaPath, FilenameFilt throws URISyntaxException, IOException { String[] files; String[] tempFiles; - List filesList = new ArrayList(); + List filesList = new ArrayList<>(); File dirFile = new File(dirURL.toURI()); tempFiles = dirFile.list(filter); @@ -418,7 +418,7 @@ protected String[] listJarResources(URL dirURL, FilenameFilter filter) URL jarFileURL = new URL(spec.substring(0, seperator)); JarFile jarFile = new JarFile(jarFileURL.toURI().getPath()); - Set filesSet = new HashSet(); + Set filesSet = new HashSet<>(); try { Enumeration entries = jarFile.entries(); @@ -486,7 +486,7 @@ protected void mergeXmlSchemas(final List storeObjects) { protected void loadFromXmlFiles() throws IOException, XMLStreamException, URISyntaxException { XMLInputFactory xmlIf = XMLInputFactory.newInstance(); - final List storeObjects = new ArrayList(); + final List storeObjects = new ArrayList<>(); xmlIf.setProperty(XMLInputFactory.IS_COALESCING, Boolean.TRUE); @@ -545,7 +545,7 @@ public boolean isLoaded() { private class StoreObjectsMerger { - private final List storeObjects = new ArrayList(); + private final List storeObjects = new ArrayList<>(); private final StoreObject targetStore = new StoreObject(); public StoreObjectsMerger(List schemaObjects) { @@ -579,7 +579,7 @@ protected void copySchemaInfo(StoreObject sourceStore) { private List createListAndFillNull(int maxIdx) { DatabaseObject[] objects = new DatabaseObject[maxIdx]; Arrays.fill(objects, null); - return new ArrayList(Arrays.asList(objects)); + return new ArrayList<>(Arrays.asList(objects)); } protected List mergeDatabaseObjects(List objects) { @@ -590,8 +590,8 @@ protected List mergeDatabaseObjects(List objects } final List orderedObjects = createListAndFillNull(maxIdx); - final List unorderedObjects = new ArrayList(); - final List mergedObjects = new ArrayList(); + final List unorderedObjects = new ArrayList<>(); + final List mergedObjects = new ArrayList<>(); for (DatabaseObject object: objects) { if (object.getOrder() > -1) { @@ -648,7 +648,7 @@ protected void validatePatch(List patches, SchemaPatch testPatch) { } protected void mergePatches(List patches) { - final List objects = new ArrayList(); + final List objects = new ArrayList<>(); Collections.sort(patches); @@ -656,7 +656,7 @@ protected void mergePatches(List patches) { validatePatch(patches, patch); objects.clear(); - List tempObjects = new ArrayList(); + List tempObjects = new ArrayList<>(); tempObjects.addAll(patch.getObjects()); patch.clearObjects(); patch.addObjects(mergeDatabaseObjects(tempObjects)); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/object/BaseSchema.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/object/BaseSchema.java index 7c437afb3b..eb715ebe6e 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/object/BaseSchema.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/object/BaseSchema.java @@ -37,7 +37,7 @@ public class BaseSchema implements Comparable { private String schemaName; @XmlElementWrapper(name="objects",namespace="http://tajo.apache.org/catalogstore") @XmlElement(name="Object",namespace="http://tajo.apache.org/catalogstore") - private final List objects = new ArrayList(); + private final List objects = new ArrayList<>(); public int getVersion() { return version; diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/object/SchemaPatch.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/object/SchemaPatch.java index 6aa450055b..ff825ae55a 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/object/SchemaPatch.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/object/SchemaPatch.java @@ -36,7 +36,7 @@ public class SchemaPatch implements Comparable { private int nextVersion; @XmlElementWrapper(name="objects",namespace="http://tajo.apache.org/catalogstore") @XmlElement(name="Object",namespace="http://tajo.apache.org/catalogstore") - private final List objects = new ArrayList(); + private final List objects = new ArrayList<>(); public int getPriorVersion() { return priorVersion; diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/object/StoreObject.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/object/StoreObject.java index c83ee3be4c..f92af99ff5 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/object/StoreObject.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/object/StoreObject.java @@ -34,13 +34,13 @@ public class StoreObject { private BaseSchema schema = new BaseSchema(); @XmlElementWrapper(name="existQueries",namespace="http://tajo.apache.org/catalogstore") @XmlElement(name="existQuery",namespace="http://tajo.apache.org/catalogstore") - private final List existQueries = new ArrayList(); + private final List existQueries = new ArrayList<>(); @XmlElementWrapper(name="dropStatements",namespace="http://tajo.apache.org/catalogstore") @XmlElement(name="dropStatement",namespace="http://tajo.apache.org/catalogstore") - private final List dropStatements = new ArrayList(); + private final List dropStatements = new ArrayList<>(); @XmlElementWrapper(name="patches",namespace="http://tajo.apache.org/catalogstore") @XmlElement(name="patch",namespace="http://tajo.apache.org/catalogstore") - private final List patches = new ArrayList(); + private final List patches = new ArrayList<>(); public BaseSchema getSchema() { return schema; diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index a5e4861dee..e4ad783d35 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -152,7 +152,7 @@ public void testCreateAndDropDatabases() throws Exception { @Test public void testCreateAndDropManyDatabases() throws Exception { - List createdDatabases = new ArrayList(); + List createdDatabases = new ArrayList<>(); InfoSchemaMetadataDictionary dictionary = new InfoSchemaMetadataDictionary(); String namePrefix = "database_"; final int NUM = 10; @@ -235,7 +235,7 @@ public void testCreateAndDropTable() throws Exception { private Map> createBaseDatabaseAndTables() throws IOException, TajoException { - Map> createdDatabaseAndTablesMap = new HashMap>(); + Map> createdDatabaseAndTablesMap = new HashMap<>(); // add and divide all tables to multiple databases in a round robin manner for (int tableId = 0; tableId < TOTAL_TABLE_NUM; tableId++) { @@ -274,7 +274,7 @@ public void testDropDatabaseWithAllTables() throws Exception { Map> createdTablesMap = createBaseDatabaseAndTables(); // Each time we drop one database, check all databases and their tables. - Iterator it = new ArrayList(createdTablesMap.keySet()).iterator(); + Iterator it = new ArrayList<>(createdTablesMap.keySet()).iterator(); while(it.hasNext()) { // drop one database String databaseName = it.next(); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/store/TestXMLCatalogSchemaManager.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/store/TestXMLCatalogSchemaManager.java index f53beffda6..a405426e04 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/store/TestXMLCatalogSchemaManager.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/store/TestXMLCatalogSchemaManager.java @@ -152,7 +152,7 @@ protected boolean matchesSafely(Iterable item, org.hamcrest.Descrip } private Matcher> hasItem(Matcher matcher) { - return new CollectionMatcher(matcher); + return new CollectionMatcher<>(matcher); } @BeforeClass @@ -193,7 +193,7 @@ public void tearDownJDBC() throws Exception { protected BaseMatcher hasItemInResultSet(final String expected, final String columnName) { return new BaseMatcher() { - private final List results = new ArrayList(); + private final List results = new ArrayList<>(); @Override public boolean matches(Object item) { diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java index 72f7f7a70a..d97b06f6be 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoAdmin.java @@ -226,12 +226,12 @@ private void processCluster(Writer writer) throws ParseException, IOException, int runningQueryMasterTasks = 0; - List liveWorkers = new ArrayList(); - List deadWorkers = new ArrayList(); - List decommissionWorkers = new ArrayList(); + List liveWorkers = new ArrayList<>(); + List deadWorkers = new ArrayList<>(); + List decommissionWorkers = new ArrayList<>(); - List liveQueryMasters = new ArrayList(); - List deadQueryMasters = new ArrayList(); + List liveQueryMasters = new ArrayList<>(); + List deadQueryMasters = new ArrayList<>(); for (WorkerResourceInfo eachWorker : workerList) { if(eachWorker.getWorkerStatus().equals(WorkerStatus.RUNNING.toString())) { diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java index a50ce7b7de..98b71a681f 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java @@ -77,7 +77,7 @@ private static Pair getConnectionAddr(TajoConf conf, CommandLin port = Integer.parseInt(conf.getVar(TajoConf.ConfVars.TAJO_MASTER_CLIENT_RPC_ADDRESS).split(":")[1]); } } - return new Pair(hostName, port); + return new Pair<>(hostName, port); } public static void main(String [] args) throws ParseException, IOException, ServiceException, SQLException { @@ -130,7 +130,7 @@ public static void dump(TajoClient client, UserRoleInfo userInfo, String baseDat if (isDumpingAllDatabases) { // sort database names in an ascending lexicographic order of the names. - List sorted = new ArrayList(client.getAllDatabaseNames()); + List sorted = new ArrayList<>(client.getAllDatabaseNames()); Collections.sort(sorted); for (String databaseName : sorted) { diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/SimpleParser.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/SimpleParser.java index 0cae7d4291..2041e729c9 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/SimpleParser.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/SimpleParser.java @@ -75,14 +75,14 @@ public static enum ParsingState { public static List parseScript(String str) throws InvalidStatementException { SimpleParser parser = new SimpleParser(); - List parsedResults = new ArrayList(); + List parsedResults = new ArrayList<>(); parsedResults.addAll(parser.parseLines(str)); parsedResults.addAll(parser.EOF()); return parsedResults; } public List parseLines(String str) throws InvalidStatementException { - List statements = new ArrayList(); + List statements = new ArrayList<>(); int lineStartIdx; int idx = 0; char [] chars = str.toCharArray(); @@ -330,7 +330,7 @@ private boolean isStatementContinue() { * @throws InvalidStatementException */ private List doProcessEndOfStatement(boolean endOfFile) throws InvalidStatementException { - List parsedResults = new ArrayList(); + List parsedResults = new ArrayList<>(); String errorMessage = ""; if (endOfFile) { if (state == ParsingState.META) { diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java index 8ae7075771..3de9e7f72e 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java @@ -91,7 +91,7 @@ public class TajoCli { TajoGetConfCommand.class, TajoHAAdminCommand.class }; - private final Map commands = new TreeMap(); + private final Map commands = new TreeMap<>(); protected static final Options options; private static final String HOME_DIR = System.getProperty("user.home"); diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java index abdbb9c6a4..b168d3f602 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescFunctionCommand.java @@ -50,7 +50,7 @@ public void invoke(String[] cmd) throws Exception { } List functions = - new ArrayList(client.getFunctions(functionName)); + new ArrayList<>(client.getFunctions(functionName)); Collections.sort(functions, new FunctionUtil.FunctionDescProtoComparator()); @@ -87,7 +87,7 @@ public void invoke(String[] cmd) throws Exception { if (printDetail && !functions.isEmpty()) { Map functionMap = - new HashMap(); + new HashMap<>(); for (CatalogProtos.FunctionDescProto eachFunction: functions) { if (!functionMap.containsKey(eachFunction.getSupplement().getShortDescription())) { diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/SetCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/SetCommand.java index 0ae07d592f..b5802a5f81 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/SetCommand.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/SetCommand.java @@ -48,7 +48,7 @@ private void showAllSessionVars() throws SQLException { } private void updateSessionVariable(String key, String val) throws NoSuchSessionVariableException { - Map variables = new HashMap(); + Map variables = new HashMap<>(); variables.put(key, val); client.updateSessionVariables(variables); } diff --git a/tajo-client/src/main/java/org/apache/tajo/client/DummyServiceTracker.java b/tajo-client/src/main/java/org/apache/tajo/client/DummyServiceTracker.java index fb2ba0c445..ed15feb091 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/DummyServiceTracker.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/DummyServiceTracker.java @@ -79,7 +79,7 @@ public int formatHA(TajoConf conf) throws ServiceTrackerException { @Override public List getMasters(TajoConf conf) throws ServiceTrackerException { - return new ArrayList(); + return new ArrayList<>(); } @Override diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java index f63bb47483..382d51ba91 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java @@ -73,7 +73,7 @@ public class SessionConnection implements Closeable { private final AtomicBoolean closed = new AtomicBoolean(false); /** session variable cache */ - private final Map sessionVarsCache = new HashMap(); + private final Map sessionVarsCache = new HashMap<>(); private final ServiceTracker serviceTracker; diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/MetaDataTuple.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/MetaDataTuple.java index bd078f6532..e96bb44e99 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/MetaDataTuple.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/MetaDataTuple.java @@ -32,10 +32,10 @@ import java.util.List; public class MetaDataTuple implements Tuple { - List values = new ArrayList(); + List values = new ArrayList<>(); public MetaDataTuple(int size) { - values = new ArrayList(size); + values = new ArrayList<>(size); for(int i = 0; i < size; i++) { values.add(NullDatum.get()); } diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 2fde31c479..ffa2dab007 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -188,7 +188,7 @@ public class QueryTestCaseBase { /** It transiently contains created tables for the running test class. */ private static String currentDatabase; - private static Set createdTableGlobalSet = new HashSet(); + private static Set createdTableGlobalSet = new HashSet<>(); // queries and results directory corresponding to subclass class. protected Path currentQueryPath; protected Path namedQueryPath; @@ -826,7 +826,7 @@ public String resultSetToString(ResultSet resultSet, boolean sort) throws SQLExc } sb.append("\n-------------------------------\n"); - List results = new ArrayList(); + List results = new ArrayList<>(); while (resultSet.next()) { StringBuilder line = new StringBuilder(); for (int i = 1; i <= numOfColumns; i++) { @@ -982,7 +982,7 @@ private List executeDDL(String ddlFileName, @Nullable String dataFileNam String compiled = compileTemplate(template, dataFilePath, args); List parsedResults = SimpleParser.parseScript(compiled); - List createdTableNames = new ArrayList(); + List createdTableNames = new ArrayList<>(); for (ParsedResult parsedResult : parsedResults) { // parse a statement @@ -1128,7 +1128,7 @@ public List listTableFiles(String tableName) throws Exception { } private List listFiles(FileSystem fs, Path path) throws Exception { - List result = new ArrayList(); + List result = new ArrayList<>(); FileStatus[] files = fs.listStatus(path); if (files == null || files.length == 0) { return result; diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 8d2ee8f9cf..3517cead5a 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -75,7 +75,7 @@ public class TajoTestingCluster { private HBaseTestClusterUtil hbaseUtil; private TajoMaster tajoMaster; - private List tajoWorkers = new ArrayList(); + private List tajoWorkers = new ArrayList<>(); private boolean isDFSRunning = false; private boolean isTajoClusterRunning = false; private boolean isCatalogServerRunning = false; diff --git a/tajo-common/src/main/java/org/apache/tajo/QueryIdFactory.java b/tajo-common/src/main/java/org/apache/tajo/QueryIdFactory.java index 9599007cae..ae05f6b916 100644 --- a/tajo-common/src/main/java/org/apache/tajo/QueryIdFactory.java +++ b/tajo-common/src/main/java/org/apache/tajo/QueryIdFactory.java @@ -36,7 +36,7 @@ public class QueryIdFactory { public static final DecimalFormat ATTEMPT_ID_FORMAT = new DecimalFormat("00"); - private static Map queryNexIdMap = new HashMap(); + private static Map queryNexIdMap = new HashMap<>(); private static AtomicInteger nextId = new AtomicInteger(0); diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java index ed84aa7f43..368e4ae16d 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/ErrorMessages.java @@ -128,7 +128,7 @@ private static void ADD_MESSAGE(ResultCode code, String msgFormat) { } private static void ADD_MESSAGE(ResultCode code, String msgFormat, int argNum) { - MESSAGES.put(code, new Pair(msgFormat, argNum)); + MESSAGES.put(code, new Pair<>(msgFormat, argNum)); } public static String getInternalErrorMessage() { diff --git a/tajo-common/src/main/java/org/apache/tajo/rule/base/BaseRuleProvider.java b/tajo-common/src/main/java/org/apache/tajo/rule/base/BaseRuleProvider.java index 2a8eed82b7..7be2eeb930 100644 --- a/tajo-common/src/main/java/org/apache/tajo/rule/base/BaseRuleProvider.java +++ b/tajo-common/src/main/java/org/apache/tajo/rule/base/BaseRuleProvider.java @@ -37,7 +37,7 @@ public class BaseRuleProvider implements SelfDiagnosisRuleProvider { public List getDefinedRules() { Set classSet = ClassUtil.findClasses(SelfDiagnosisRule.class, getClass().getPackage().getName()); - List ruleList = new ArrayList(classSet.size()); + List ruleList = new ArrayList<>(classSet.size()); for (Class ruleClazz: classSet) { try { diff --git a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeConstants.java b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeConstants.java index 1f77f430aa..e610deeacd 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeConstants.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeConstants.java @@ -610,7 +610,7 @@ public TokenField getValueType() { return valueType; } } - public static final Map dateTokenMap = new HashMap(); + public static final Map dateTokenMap = new HashMap<>(); static { for (Object[] eachToken: datetktbl) { diff --git a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeFormat.java b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeFormat.java index 798b9c5664..19ad06547f 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeFormat.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/datetime/DateTimeFormat.java @@ -469,7 +469,7 @@ enum FromCharDateMode static final KeyWord[] DCH_keywords = new KeyWord[DCH_keywordValues.length]; - static Map DCH_index = new HashMap(); + static Map DCH_index = new HashMap<>(); static { int index = 0; @@ -546,7 +546,7 @@ static class TmFromChar { int yysz; /* is it YY or YYYY ? */ int clock; /* 12 or 24 hour clock? */ } - static Map formatNodeCache = new HashMap(); + static Map formatNodeCache = new HashMap<>(); /** * ---------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/graph/DirectedGraphCursor.java b/tajo-common/src/main/java/org/apache/tajo/util/graph/DirectedGraphCursor.java index 9b15cfe466..7d25dc8d36 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/graph/DirectedGraphCursor.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/graph/DirectedGraphCursor.java @@ -22,7 +22,7 @@ public class DirectedGraphCursor { private DirectedGraph graph; - private ArrayList orderedVertices = new ArrayList(); + private ArrayList orderedVertices = new ArrayList<>(); private int cursor = 0; public DirectedGraphCursor(DirectedGraph graph, V startVertex) { diff --git a/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java b/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java index 045add57fb..d3a3a1e693 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java @@ -153,7 +153,7 @@ public List getOutgoingEdges(V tail) { @Override public List getChilds(V v) { - List childBlocks = new ArrayList(); + List childBlocks = new ArrayList<>(); if (reversedEdges.containsKey(v)) { for (Map.Entry entry: reversedEdges.get(v).entrySet()) { childBlocks.add(entry.getKey()); @@ -190,7 +190,7 @@ public V getChild(V block, int idx) { @Override public List getParents(V block) { - List childBlocks = new ArrayList(); + List childBlocks = new ArrayList<>(); if (directedEdges.containsKey(block)) { for (Map.Entry entry: directedEdges.get(block).entrySet()) { childBlocks.add(entry.getKey()); @@ -220,7 +220,7 @@ public int getParentCount(V block) { @Override public void accept(V source, DirectedGraphVisitor visitor) { - Stack stack = new Stack(); + Stack stack = new Stack<>(); visitRecursive(stack, source, visitor); } @@ -267,7 +267,7 @@ private class DepthString { } private class QueryGraphTopologyStringBuilder implements DirectedGraphVisitor { - Stack depthString = new Stack(); + Stack depthString = new Stack<>(); @Override public void visit(Stack stack, V vertex) { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java index cdcdb6762d..c497d74b9a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestDefaultCliOutputFormatter.java @@ -144,7 +144,7 @@ public void testPrintResultSelectStatement() throws Exception { stats.setNumRows(numRows); tableDesc.setStats(stats); - final List resultTables = new ArrayList(); + final List resultTables = new ArrayList<>(); String expectedOutput = "col1, col2, col3\n"; expectedOutput += "-------------------------------\n"; diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java index 60f04db84f..279e0a4980 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -694,7 +694,7 @@ public void testNullCharSessionInCTAS() throws Exception { " c_custkey,\n" + " orders.o_orderkey;\n"; - Map variables = new HashMap(); + Map variables = new HashMap<>(); variables.put(SessionVars.NULL_CHAR.keyname(), "\\\\T"); client.updateSessionVariables(variables); ResultSet res = client.executeQueryAndGetResult(sql); @@ -763,7 +763,7 @@ public void testGetQueryInfoAndHistory() throws Exception { assertEquals(2, queryHistory.getStageHistoriesCount()); List taskHistories = - new ArrayList(queryHistory.getStageHistoriesList()); + new ArrayList<>(queryHistory.getStageHistoriesList()); Collections.sort(taskHistories, new Comparator() { @Override public int compare(ClientProtos.StageHistoryProto o1, StageHistoryProto o2) { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java index fd8e502402..7bba76b626 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java @@ -347,7 +347,7 @@ public final void testFindDistinctAggFunctions() throws TajoException { GroupbyNode groupByNode = plan.getRootBlock().getNode(NodeType.GROUP_BY); EvalNode [] aggEvals = groupByNode.getAggFunctions(); - List list = new ArrayList(); + List list = new ArrayList<>(); for (int i = 0; i < aggEvals.length; i++) { list.addAll(EvalTreeUtil.findDistinctAggFunction(aggEvals[i])); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java index a03ee492e6..24feee0cec 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java @@ -73,11 +73,11 @@ public final void testQueryBlockGraph() { assertTrue(graph.isLeaf(new1.getName())); assertTrue(graph.isLeaf(new2.getName())); - Set result = new HashSet(); + Set result = new HashSet<>(); result.add(new1); result.add(new2); - Set childs = new HashSet(plan.getChildBlocks(root)); + Set childs = new HashSet<>(plan.getChildBlocks(root)); assertEquals(result, childs); assertEquals(root, plan.getParentBlock(new1)); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 38d02aa5f2..e9220c332e 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -865,7 +865,7 @@ public final void testVisitor() throws TajoException { } private static class TestVisitor implements LogicalNodeVisitor { - Stack stack = new Stack(); + Stack stack = new Stack<>(); @Override public void visit(LogicalNode node) { stack.push(node); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java index 5d02a7ff1d..c9e0c46225 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java @@ -342,7 +342,7 @@ public void testGetNonZeroLengthDataFiles() throws Exception { FileSystem fs = path.getFileSystem(util.getConfiguration()); - List expectedFiles = new ArrayList(); + List expectedFiles = new ArrayList<>(); RemoteIterator files = fs.listFiles(path, true); while (files.hasNext()) { LocatedFileStatus file = files.next(); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashPartitioner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashPartitioner.java index 1146e853e5..9558b2d706 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashPartitioner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestHashPartitioner.java @@ -102,7 +102,7 @@ public final void testGetPartition2() { for (int index = 0; index < testNumPartitions.length; index++) { Partitioner p = new HashPartitioner(new int[]{0, 1, 2}, testNumPartitions[index]); - Set ids = new TreeSet(); + Set ids = new TreeSet<>(); for (int i = 0; i < data.length; i++) { Tuple tuple = new VTuple( new Datum[]{new TextDatum(data[i][0]), new TextDatum(data[i][1]), new TextDatum(data[i][2])}); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 0b94c52f96..8bc619c17b 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -662,7 +662,7 @@ public final void testPartitionedStorePlan() throws IOException, TajoException { Path queryLocalTmpDir = new Path(conf.getVar(ConfVars.WORKER_TEMPORAL_DIR) + "/" + executionBlockBaseDir); FileStatus [] list = fs.listStatus(queryLocalTmpDir); - List fragments = new ArrayList(); + List fragments = new ArrayList<>(); for (FileStatus status : list) { assertTrue(status.isDirectory()); FileStatus [] files = fs.listStatus(status.getPath()); @@ -796,7 +796,7 @@ public final void testPartitionedStorePlanWithEmptyGroupingSet() Path queryLocalTmpDir = new Path(conf.getVar(ConfVars.WORKER_TEMPORAL_DIR) + "/" + executionBlockBaseDir); FileStatus [] list = fs.listStatus(queryLocalTmpDir); - List fragments = new ArrayList(); + List fragments = new ArrayList<>(); for (FileStatus status : list) { assertTrue(status.isDirectory()); FileStatus [] files = fs.listStatus(status.getPath()); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java index 753eebc57d..2f7330b09d 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java @@ -76,7 +76,7 @@ public final void testSortBench() { for(int iteration = 0; iteration < ITERATION; iteration++) { TupleList target = new TupleList(tuples.length); target.addAll(Arrays.asList(Arrays.copyOf(tuples, tuples.length))); - Set keys = new TreeSet(); + Set keys = new TreeSet<>(); for (int i = 0; i < MAX_SORT_KEY; i++) { keys.add(rnd.nextInt(schema.size())); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java index 0fe5e0ea38..dd321ae9ca 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java @@ -49,7 +49,7 @@ public class TestGroupByQuery extends QueryTestCaseBase { public TestGroupByQuery(String groupByOption) throws Exception { super(TajoConstants.DEFAULT_DATABASE_NAME); - Map variables = new HashMap(); + Map variables = new HashMap<>(); if (groupByOption.equals("MultiLevel")) { variables.put(SessionVars.GROUPBY_MULTI_LEVEL_ENABLED.keyname(), "true"); } else { @@ -698,14 +698,14 @@ public final void testNumShufflePartition() throws Exception { schema.addColumn("col1", Type.TEXT); schema.addColumn("col2", Type.TEXT); - List data = new ArrayList(); + List data = new ArrayList<>(); int totalBytes = 0; Random rand = new Random(System.currentTimeMillis()); String col1Prefix = "Column-1Column-1Column-1Column-1Column-1Column-1Column-1Column-1Column-1Column-1Column-1" + "Column-1Column-1Column-1Column-1Column-1Column-1Column-1Column-1Column-1Column-1Column-1" + "Column-1Column-1Column-1Column-1Column-1Column-1Column-1Column-1Column-1Column-1Column-1"; - Set uniqKeys = new HashSet(); + Set uniqKeys = new HashSet<>(); while(true) { int col1RandomValue = rand.nextInt(1000000); uniqKeys.add(col1RandomValue); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java index 59cb3ec57a..0d9c49b907 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestHBaseTable.java @@ -789,7 +789,7 @@ public void testInsertIntoMultiRegion() throws Exception { Schema schema = new Schema(); schema.addColumn("id", Type.TEXT); schema.addColumn("name", Type.TEXT); - List datas = new ArrayList(); + List datas = new ArrayList<>(); DecimalFormat df = new DecimalFormat("000"); for (int i = 99; i >= 0; i--) { datas.add(df.format(i) + "|value" + i); @@ -845,7 +845,7 @@ public void testInsertIntoMultiRegion2() throws Exception { Schema schema = new Schema(); schema.addColumn("id", Type.TEXT); schema.addColumn("name", Type.TEXT); - List datas = new ArrayList(); + List datas = new ArrayList<>(); for (int i = 99; i >= 0; i--) { datas.add(i + "|value" + i); } @@ -903,7 +903,7 @@ public void testInsertIntoMultiRegionWithSplitFile() throws Exception { Schema schema = new Schema(); schema.addColumn("id", Type.TEXT); schema.addColumn("name", Type.TEXT); - List datas = new ArrayList(); + List datas = new ArrayList<>(); DecimalFormat df = new DecimalFormat("000"); for (int i = 99; i >= 0; i--) { datas.add(df.format(i) + "|value" + i); @@ -963,7 +963,7 @@ public void testInsertIntoMultiRegionMultiRowFields() throws Exception { schema.addColumn("id2", Type.TEXT); schema.addColumn("name", Type.TEXT); DecimalFormat df = new DecimalFormat("000"); - List datas = new ArrayList(); + List datas = new ArrayList<>(); for (int i = 99; i >= 0; i--) { datas.add(df.format(i) + "|" + (i + 100) + "|value" + i); } @@ -1018,7 +1018,7 @@ public void testInsertIntoBinaryMultiRegion() throws Exception { Schema schema = new Schema(); schema.addColumn("id", Type.INT4); schema.addColumn("name", Type.TEXT); - List datas = new ArrayList(); + List datas = new ArrayList<>(); for (int i = 99; i >= 0; i--) { datas.add(i + "|value" + i); } @@ -1076,7 +1076,7 @@ public void testInsertIntoColumnKeyValue() throws Exception { schema.addColumn("col2_key", Type.TEXT); schema.addColumn("col2_value", Type.TEXT); schema.addColumn("col3", Type.TEXT); - List datas = new ArrayList(); + List datas = new ArrayList<>(); for (int i = 20; i >= 0; i--) { for (int j = 0; j < 3; j++) { datas.add(i + "|ck-" + j + "|value-" + j + "|col3-" + i); @@ -1163,7 +1163,7 @@ public void testInsertIntoDifferentType() throws Exception { Schema schema = new Schema(); schema.addColumn("id", Type.INT4); schema.addColumn("name", Type.TEXT); - List datas = new ArrayList(); + List datas = new ArrayList<>(); for (int i = 99; i >= 0; i--) { datas.add(i + "|value" + i); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java index a255c6dd40..19b1486c20 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestIndexScan.java @@ -37,7 +37,7 @@ public class TestIndexScan extends QueryTestCaseBase { public TestIndexScan() throws ServiceException, SQLException, NoSuchSessionVariableException { super(TajoConstants.DEFAULT_DATABASE_NAME); - Map sessionVars = new HashMap(); + Map sessionVars = new HashMap<>(); sessionVars.put(SessionVars.INDEX_ENABLED.keyname(), "true"); sessionVars.put(SessionVars.INDEX_SELECTIVITY_THRESHOLD.keyname(), "0.01f"); client.updateSessionVariables(sessionVars); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java index 90a004b324..2372a11a47 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinQuery.java @@ -296,7 +296,7 @@ protected static void addEmptyDataFile(String tableName, boolean isPartitioned) protected static List getPartitionPathList(FileSystem fs, Path path) throws Exception { FileStatus[] files = fs.listStatus(path); - List paths = new ArrayList(); + List paths = new ArrayList<>(); if (files != null) { for (FileStatus eachFile: files) { if (eachFile.isFile()) { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJsonWithTimezone.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJsonWithTimezone.java index e526f5c62b..3380b1fc59 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJsonWithTimezone.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJsonWithTimezone.java @@ -62,7 +62,7 @@ public void testTimezonedTable3() throws Exception { // Table - timezone = GMT+9 // Client - GMT+9 through TajoClient API - Map sessionVars = new HashMap(); + Map sessionVars = new HashMap<>(); sessionVars.put(SessionVars.TIMEZONE.name(), "GMT+9"); getClient().updateSessionVariables(sessionVars); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 130b02d22d..e55acf1fc0 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -613,7 +613,7 @@ public void testTimezonedTable3() throws Exception { // Table - timezone = GMT+9 // Client - GMT+9 through TajoClient API - Map sessionVars = new HashMap(); + Map sessionVars = new HashMap<>(); sessionVars.put(SessionVars.TIMEZONE.name(), "GMT+9"); getClient().updateSessionVariables(sessionVars); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java index 94e5e71602..5b495071bf 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java @@ -1091,7 +1091,7 @@ public void testScatteredHashShuffle() throws Exception { schema.addColumn("col1", TajoDataTypes.Type.TEXT); schema.addColumn("col2", TajoDataTypes.Type.TEXT); - List data = new ArrayList(); + List data = new ArrayList<>(); int totalBytes = 0; Random rand = new Random(System.currentTimeMillis()); String col2Data = "Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2Column-2" + diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/util/TestTableCache.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/util/TestTableCache.java index f10f2a1354..2161074640 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/util/TestTableCache.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/util/TestTableCache.java @@ -50,7 +50,7 @@ public void testBroadcastTableCache() throws Exception { final int parallelCount = 30; ExecutorService executor = Executors.newFixedThreadPool(parallelCount); - List>> tasks = new ArrayList>>(); + List>> tasks = new ArrayList<>(); for (int i = 0; i < parallelCount; i++) { tasks.add(executor.submit(createTask(key, resource))); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestRepartitioner.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestRepartitioner.java index 36942a40f9..c1461b0668 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestRepartitioner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestRepartitioner.java @@ -54,7 +54,7 @@ public void testCreateHashFetchURL() throws Exception { ExecutionBlockId sid = new ExecutionBlockId(q1, 2); int numPartition = 10; - Map> intermediateEntries = new HashMap>(); + Map> intermediateEntries = new HashMap<>(); for (int i = 0; i < numPartition; i++) { intermediateEntries.put(i, new ArrayList()); } @@ -67,7 +67,7 @@ public void testCreateHashFetchURL() throws Exception { } Map>> hashEntries = - new HashMap>>(); + new HashMap<>(); for (Map.Entry> eachEntry: intermediateEntries.entrySet()) { FetchImpl fetch = new FetchImpl(new Task.PullHost(hostName, port), ShuffleType.HASH_SHUFFLE, @@ -79,7 +79,7 @@ public void testCreateHashFetchURL() throws Exception { fetch = new FetchImpl(proto); assertEquals(proto, fetch.getProto()); - Map> ebEntries = new HashMap>(); + Map> ebEntries = new HashMap<>(); ebEntries.put(sid, eachEntry.getValue()); hashEntries.put(eachEntry.getKey(), ebEntries); @@ -172,12 +172,12 @@ private static void assertFetchVolumes(long [] expected, Long [] results) { @Test public void testMergeIntermediates() { //Test Merge - List intermediateEntries = new ArrayList(); + List intermediateEntries = new ArrayList<>(); int[] pageLengths = {10 * 1024 * 1024, 10 * 1024 * 1024, 10 * 1024 * 1024, 5 * 1024 * 1024}; //35 MB long expectedTotalLength = 0; for (int i = 0; i < 20; i++) { - List> pages = new ArrayList>(); + List> pages = new ArrayList<>(); long offset = 0; for (int j = 0; j < pageLengths.length; j++) { pages.add(new Pair(offset, pageLengths[j])); @@ -222,7 +222,7 @@ public void testMergeIntermediates() { @Test public void testSplitIntermediates() { - List intermediateEntries = new ArrayList(); + List intermediateEntries = new ArrayList<>(); int[] pageLengths = new int[20]; //195MB for (int i = 0 ; i < pageLengths.length; i++) { @@ -235,7 +235,7 @@ public void testSplitIntermediates() { long expectedTotalLength = 0; for (int i = 0; i < 20; i++) { - List> pages = new ArrayList>(); + List> pages = new ArrayList<>(); long offset = 0; for (int j = 0; j < pageLengths.length; j++) { pages.add(new Pair(offset, pageLengths[j])); @@ -256,7 +256,7 @@ public void testSplitIntermediates() { int index = 0; int numZeroPosFetcher = 0; long totalLength = 0; - Set uniqPullHost = new HashSet(); + Set uniqPullHost = new HashSet<>(); for (List eachFetchList: fetches) { long length = 0; @@ -365,9 +365,9 @@ public void testSplitIntermediates2() { {844591486, 5523957} }; - List entries = new ArrayList(); + List entries = new ArrayList<>(); for (int i = 0; i < 2; i++) { - List> pages = new ArrayList>(); + List> pages = new ArrayList<>(); for (int j = 0; j < pageDatas.length; j++) { pages.add(new Pair(pageDatas[j][0], (int) (pageDatas[j][1]))); } @@ -409,7 +409,7 @@ public void testSplitIntermediates2() { @Test public void testSplitIntermediatesWithUniqueHost() { - List intermediateEntries = new ArrayList(); + List intermediateEntries = new ArrayList<>(); int[] pageLengths = new int[20]; //195MB for (int i = 0 ; i < pageLengths.length; i++) { @@ -424,7 +424,7 @@ public void testSplitIntermediatesWithUniqueHost() { Task.PullHost pullHost = new Task.PullHost("host", 0); for (int i = 0; i < 20; i++) { - List> pages = new ArrayList>(); + List> pages = new ArrayList<>(); long offset = 0; for (int j = 0; j < pageLengths.length; j++) { pages.add(new Pair(offset, pageLengths[j])); @@ -454,7 +454,7 @@ public void testSplitIntermediatesWithUniqueHost() { int index = 0; int numZeroPosFetcher = 0; long totalLength = 0; - Set uniqPullHost = new HashSet(); + Set uniqPullHost = new HashSet<>(); for (List eachFetchList: fetches) { long length = 0; diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/scheduler/TestSimpleScheduler.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/scheduler/TestSimpleScheduler.java index 328c28141a..89fdba0459 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/master/scheduler/TestSimpleScheduler.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/scheduler/TestSimpleScheduler.java @@ -179,7 +179,7 @@ public void testReserveResource() throws InterruptedException, ExecutionExceptio assertEquals(totalResource, scheduler.getClusterResource()); QueryId queryId = QueryIdFactory.newQueryId(System.nanoTime(), 0); - CallFuture callBack = new CallFuture(); + CallFuture callBack = new CallFuture<>(); rmContext.getDispatcher().getEventHandler().handle(new ResourceReserveSchedulerEvent( createResourceRequest(queryId, requestNum, new ArrayList()), callBack)); @@ -210,7 +210,7 @@ public void testReserveResourceWithWorkerPriority() throws InterruptedException, QueryId queryId = QueryIdFactory.newQueryId(System.nanoTime(), 0); NodeResourceRequest requestProto = createResourceRequest(queryId, requestNum, targetWorkers); - CallFuture callBack = new CallFuture(); + CallFuture callBack = new CallFuture<>(); rmContext.getDispatcher().getEventHandler().handle(new ResourceReserveSchedulerEvent( requestProto, callBack)); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestIntermediateEntry.java b/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestIntermediateEntry.java index 237fb32ff2..9e134cdcb1 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestIntermediateEntry.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestIntermediateEntry.java @@ -31,7 +31,7 @@ public class TestIntermediateEntry { public void testPage() { Task.IntermediateEntry interm = new Task.IntermediateEntry(-1, -1, 1, null); - List> pages = new ArrayList>(); + List> pages = new ArrayList<>(); pages.add(new Pair(0L, 1441275)); pages.add(new Pair(1441275L, 1447446)); pages.add(new Pair(2888721L, 1442507)); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/util/TestJSPUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestJSPUtil.java index f3b6936dd1..3711f4f12e 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/util/TestJSPUtil.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/util/TestJSPUtil.java @@ -35,7 +35,7 @@ public class TestJSPUtil { @Test public void testSortTask() throws Exception { - List tasks = new ArrayList(); + List tasks = new ArrayList<>(); Configuration conf = new TajoConf(); @@ -84,7 +84,7 @@ public void testSortTask() throws Exception { @Test public void testGetPageNavigationList() { - List originList = new ArrayList(); + List originList = new ArrayList<>(); for (int i = 0; i < 35; i++) { originList.add("Data" + (i + 1)); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/util/history/TestHistoryWriterReader.java b/tajo-core-tests/src/test/java/org/apache/tajo/util/history/TestHistoryWriterReader.java index 3d2578c0f3..8b33d02acd 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/util/history/TestHistoryWriterReader.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/util/history/TestHistoryWriterReader.java @@ -195,14 +195,14 @@ public void testQueryHistoryReadAndWrite() throws Exception { QueryId queryId = QueryIdFactory.newQueryId(startTime, 1); queryHistory.setQueryId(queryId.toString()); queryHistory.setLogicalPlan("LogicalPlan"); - List stages = new ArrayList(); + List stages = new ArrayList<>(); for (int i = 0; i < 3; i++) { ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(queryId, i); StageHistory stageHistory = new StageHistory(); stageHistory.setExecutionBlockId(ebId.toString()); stageHistory.setStartTime(startTime + i); - List taskHistories = new ArrayList(); + List taskHistories = new ArrayList<>(); for (int j = 0; j < 5; j++) { TaskHistory taskHistory = new TaskHistory(); taskHistory.setId(QueryIdFactory.newTaskAttemptId(QueryIdFactory.newTaskId(ebId), 1).toString()); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/util/metrics/TestMetricsFilter.java b/tajo-core-tests/src/test/java/org/apache/tajo/util/metrics/TestMetricsFilter.java index b70512c8c1..9c52bb199f 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/util/metrics/TestMetricsFilter.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/util/metrics/TestMetricsFilter.java @@ -38,7 +38,7 @@ public void testGroupNameMetricsFilter() { @Test public void testRegexpMetricsFilter() { - List filterExpressions = new ArrayList(); + List filterExpressions = new ArrayList<>(); filterExpressions.add("JVM"); filterExpressions.add("Query"); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java b/tajo-core-tests/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java index 8751df9054..836136879d 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java @@ -125,7 +125,7 @@ public void testMetricsReporter() throws Exception { String line; - List lines = new ArrayList(); + List lines = new ArrayList<>(); while((line = reader.readLine()) != null) { lines.add(line); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/worker/MockNodeResourceManager.java b/tajo-core-tests/src/test/java/org/apache/tajo/worker/MockNodeResourceManager.java index 8c8427d37b..786498e008 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/worker/MockNodeResourceManager.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/worker/MockNodeResourceManager.java @@ -67,7 +67,7 @@ protected static Queue createTaskRequests( ExecutionBlockId ebId, int memory, int size) { Queue - requestProtoList = new LinkedBlockingQueue(); + requestProtoList = new LinkedBlockingQueue<>(); for (int i = 0; i < size; i++) { TaskAttemptId taskAttemptId = QueryIdFactory.newTaskAttemptId(QueryIdFactory.newTaskId(ebId, i), 0); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java b/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java index 1193478024..3d0554ae40 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java @@ -141,7 +141,7 @@ public void testNodeResourceAllocateEvent() throws Exception { int requestSize = 4; resourceManager.setTaskHandlerEvent(false); //skip task execution - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0); requestProto.setExecutionBlockId(ebId.getProto()); @@ -164,7 +164,7 @@ public void testNodeResourceCancellation() throws Exception { int overSize = 10; resourceManager.setTaskHandlerEvent(false); //skip task execution - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0); requestProto.setExecutionBlockId(ebId.getProto()); @@ -184,7 +184,7 @@ public void testNodeResourceDeallocateEvent() throws Exception { int requestSize = 4; resourceManager.setTaskHandlerEvent(false); //skip task execution - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0); requestProto.setExecutionBlockId(ebId.getProto()); @@ -226,7 +226,7 @@ public void testParallelRequest() throws Exception { BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); requestProto.addTaskRequest(task); requestProto.setExecutionBlockId(ebId.getProto()); - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture)); assertTrue(callFuture.get().getCancellationTaskCount() == 0); totalComplete.incrementAndGet(); @@ -251,7 +251,7 @@ public void run() { requestProto.addTaskRequest(task); requestProto.setExecutionBlockId(ebId.getProto()); - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture)); try { BatchAllocationResponse proto = callFuture.get(); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java b/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java index 7cb7e7129f..5039c8beef 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java @@ -143,7 +143,7 @@ public void testTaskRequest() throws Exception { QueryId qid = LocalTajoTestingUtility.newQueryId(); ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(qid, 1); - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); requestProto.setExecutionBlockId(ebId.getProto()); @@ -171,7 +171,7 @@ public void testTaskException() throws Exception { QueryId qid = LocalTajoTestingUtility.newQueryId(); ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(qid, 1); - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); requestProto.setExecutionBlockId(ebId.getProto()); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestTaskManager.java b/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestTaskManager.java index 1d84afe59f..c19d8fa1c1 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestTaskManager.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/worker/TestTaskManager.java @@ -147,7 +147,7 @@ public void testExecutionBlockStart() throws Exception { QueryId qid = LocalTajoTestingUtility.newQueryId(); ExecutionBlockId ebId = QueryIdFactory.newExecutionBlockId(qid, 1); - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); requestProto.setExecutionBlockId(ebId.getProto()); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ws/rs/resources/TestSessionsResource.java b/tajo-core-tests/src/test/java/org/apache/tajo/ws/rs/resources/TestSessionsResource.java index 7b293d840e..99a6d620c5 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ws/rs/resources/TestSessionsResource.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ws/rs/resources/TestSessionsResource.java @@ -195,10 +195,10 @@ public void testUpdateSessionVariables() throws Exception { assertNotNull(response); assertTrue(response.getId() != null && !response.getId().isEmpty()); - Map variablesMap = new HashMap(); + Map variablesMap = new HashMap<>(); variablesMap.put("variableA", "valueA"); variablesMap.put("variableB", "valueB"); - Map> variables = new HashMap>(); + Map> variables = new HashMap<>(); variables.put("variables", variablesMap); Response restResponse = restClient.target(sessionsURI) .path("/{session-id}/variables").resolveTemplate("session-id", response.getId()) @@ -236,7 +236,7 @@ public void testUpdateSessionVariable() throws Exception { assertNotNull(response); assertTrue(response.getId() != null && !response.getId().isEmpty()); - Map variablesMap = new HashMap(); + Map variablesMap = new HashMap<>(); variablesMap.put("variableA", "valueA"); Response restResponse = restClient.target(sessionsURI) .path("/{session-id}/variables").resolveTemplate("session-id", response.getId()) diff --git a/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java b/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java index d2f56d3291..f4527f02d0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java +++ b/tajo-core/src/main/java/org/apache/tajo/benchmark/BenchmarkSet.java @@ -41,9 +41,9 @@ public abstract class BenchmarkSet { protected TajoClient tajo; - protected Map schemas = new HashMap(); - protected Map outSchemas = new HashMap(); - protected Map queries = new HashMap(); + protected Map schemas = new HashMap<>(); + protected Map outSchemas = new HashMap<>(); + protected Map queries = new HashMap<>(); protected String dataDir; public void init(TajoConf conf, String dataDir) throws SQLException { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java index fbd88addba..adf83ba859 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java @@ -75,7 +75,7 @@ public Map, EvalNode> getPrecompiedEvals() { } private static void compileIfAbsent(CompilationContext context, Schema schema, EvalNode eval) { - Pair key = new Pair(schema, eval); + Pair key = new Pair<>(schema, eval); if (!context.compiledEval.containsKey(key)) { try { EvalNode compiled = context.compiler.compile(schema, eval); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java index 6c3bf17c7d..b1a15ae9ee 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java @@ -819,7 +819,7 @@ public void newArray(final Class clazz) { private int nextVarId = 3; - private Map localVariablesMap = new HashMap(); + private Map localVariablesMap = new HashMap<>(); public void astore(String name) { if (localVariablesMap.containsKey(name)) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java index b765f5ee07..5d172daeb0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionLoader.java @@ -229,7 +229,7 @@ private static StaticMethodInvocationDesc extractStaticMethodInvocation(Method m * @return A list of FunctionDescs */ public static List findLegacyFunctions() { - List sqlFuncs = new ArrayList(); + List sqlFuncs = new ArrayList<>(); Set functionClasses = ClassUtil.findClasses(Function.class, "org.apache.tajo.engine.function"); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Lead.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Lead.java index 8edf1fb48e..5f3290c97d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Lead.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Lead.java @@ -85,7 +85,7 @@ public Datum terminate(FunctionContext ctx) { } private static class LeadContext implements FunctionContext { - LinkedList leadBuffer = new LinkedList(); + LinkedList leadBuffer = new LinkedList<>(); int leadNum = -1; Datum defaultDatum = null; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 461e4055bb..74f978b983 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -1095,7 +1095,7 @@ private SortExec createSortExecForDistinctGroupby(TaskAttemptContext context, SortNode sortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); //2 phase: seq, groupby columns, distinct1 keys, distinct2 keys, //3 phase: groupby columns, seq, distinct1 keys, distinct2 keys, - List sortSpecs = new ArrayList(); + List sortSpecs = new ArrayList<>(); if (phase == 2) { sortSpecs.add(new SortSpec(distinctNode.getTargets()[0].getNamedColumn())); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java index c71324d576..6e7720f7f8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java @@ -32,11 +32,11 @@ public class ExecutionBlock { private ExecutionBlockId executionBlockId; private LogicalNode plan = null; private StoreTableNode store = null; - private List scanlist = new ArrayList(); + private List scanlist = new ArrayList<>(); private Enforcer enforcer = new Enforcer(); // Actual ScanNode's ExecutionBlockId -> Delegated ScanNode's ExecutionBlockId. - private Map unionScanMap = new HashMap(); + private Map unionScanMap = new HashMap<>(); private boolean hasJoinPlan; private boolean hasUnionPlan; @@ -111,7 +111,7 @@ public void setPlan(LogicalNode plan) { } LogicalNode node = plan; - ArrayList s = new ArrayList(); + ArrayList s = new ArrayList<>(); s.add(node); while (!s.isEmpty()) { node = s.remove(s.size()-1); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java index c6864b9a0c..10f6e54722 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlockCursor.java @@ -26,11 +26,11 @@ */ public class ExecutionBlockCursor implements Iterable { private MasterPlan masterPlan; - private ArrayList orderedBlocks = new ArrayList(); + private ArrayList orderedBlocks = new ArrayList<>(); - private List executionOrderedBlocks = new ArrayList(); - private List notOrderedSiblingBlocks = new ArrayList(); - private Map orderRequiredChildCountMap = new HashMap(); + private List executionOrderedBlocks = new ArrayList<>(); + private List notOrderedSiblingBlocks = new ArrayList<>(); + private Map orderRequiredChildCountMap = new HashMap<>(); public ExecutionBlockCursor(MasterPlan plan) { this(plan, false); @@ -149,7 +149,7 @@ In the case of the upper plan, buildDepthFirstOrder() makes the following order } private void preExecutionOrder(BuildOrderItem current) { - Stack stack = new Stack(); + Stack stack = new Stack<>(); if (!masterPlan.isLeaf(current.eb.getId())) { List children = masterPlan.getChilds(current.eb); orderRequiredChildCountMap.put(current.eb.getId(), new AtomicInteger(children.size())); @@ -172,7 +172,7 @@ private void preExecutionOrder(BuildOrderItem current) { class BuildOrderItem { ExecutionBlock eb; ExecutionBlock parentEB; - List siblings = new ArrayList(); + List siblings = new ArrayList<>(); BuildOrderItem(ExecutionBlock parentEB, ExecutionBlock eb) { this.parentEB = parentEB; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 504999567f..33277b0d9d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -1277,7 +1277,7 @@ public LogicalNode visitTableSubQuery(GlobalPlanContext context, LogicalPlan pla ExecutionBlock currentBlock = context.execBlockMap.remove(child.getPID()); if (child.getType() == NodeType.UNION) { - List addedTableSubQueries = new ArrayList(); + List addedTableSubQueries = new ArrayList<>(); TableSubQueryNode leftMostSubQueryNode = null; for (ExecutionBlock childBlock : context.plan.getChilds(currentBlock.getId())) { TableSubQueryNode copy = PlannerUtil.clone(plan, node); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java index 8a7229b3e2..90ccc3a41a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java @@ -43,9 +43,9 @@ public class MasterPlan { private AtomicInteger nextId = new AtomicInteger(0); private ExecutionBlock terminalBlock; - private Map execBlockMap = new HashMap(); + private Map execBlockMap = new HashMap<>(); private SimpleDirectedGraph execBlockGraph = - new SimpleDirectedGraph(); + new SimpleDirectedGraph<>(); public ExecutionBlockId newExecutionBlockId() { return new ExecutionBlockId(queryId, nextId.incrementAndGet()); @@ -196,7 +196,7 @@ public List getChilds(ExecutionBlock execBlock) { } public List getChilds(ExecutionBlockId id) { - List childBlocks = new ArrayList(); + List childBlocks = new ArrayList<>(); for (ExecutionBlockId cid : execBlockGraph.getChilds(id)) { childBlocks.add(execBlockMap.get(cid)); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ParallelExecutionQueue.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ParallelExecutionQueue.java index 1e823be9e0..70397ff9a7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ParallelExecutionQueue.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ParallelExecutionQueue.java @@ -35,7 +35,7 @@ public class ParallelExecutionQueue implements ExecutionQueue, Iterable> executable; - private final Set executed = new HashSet(); + private final Set executed = new HashSet<>(); public ParallelExecutionQueue(MasterPlan masterPlan, int maximum) { this.masterPlan = masterPlan; @@ -44,7 +44,7 @@ public ParallelExecutionQueue(MasterPlan masterPlan, int maximum) { } private List> toStacks(ExecutionBlock root) { - List> stacks = new ArrayList>(); + List> stacks = new ArrayList<>(); toStacks(root, stacks, new ArrayList()); return stacks; } @@ -54,7 +54,7 @@ private void toStacks(ExecutionBlock current, List> queues List stack) { stack.add(current); if (masterPlan.isLeaf(current.getId())) { - queues.add(new ArrayDeque(stack)); + queues.add(new ArrayDeque<>(stack)); } else { List children = masterPlan.getChilds(current); for (int i = 0; i < children.size(); i++) { @@ -75,7 +75,7 @@ public synchronized int size() { @Override public synchronized ExecutionBlock[] first() { int max = Math.min(maximum, executable.size()); - List result = new ArrayList(); + List result = new ArrayList<>(); for (Deque queue : executable) { if (result.size() < max && isExecutableNow(queue.peekLast())) { result.add(queue.removeLast()); @@ -106,7 +106,7 @@ private boolean isExecutableNow(ExecutionBlock current) { List dependents = masterPlan.getChilds(current); if (parent != null && masterPlan.getChannel(current.getId(), parent.getId()).needShuffle()) { // add all children of sibling for partitioning - dependents = new ArrayList(); + dependents = new ArrayList<>(); for (ExecutionBlock sibling : masterPlan.getChilds(parent)) { dependents.addAll(masterPlan.getChilds(sibling)); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index 6fbecc1d85..459d9af8f0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -197,13 +197,13 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou */ List originalGroupingColumns = Arrays.asList(groupbyNode.getGroupingColumns()); - List childGroupbyNodes = new ArrayList(); + List childGroupbyNodes = new ArrayList<>(); - List otherAggregationFunctionCallEvals = new ArrayList(); - List otherAggregationFunctionTargets = new ArrayList(); + List otherAggregationFunctionCallEvals = new ArrayList<>(); + List otherAggregationFunctionTargets = new ArrayList<>(); //distinct columns -> GroupbyNode - Map distinctNodeBuildInfos = new HashMap(); + Map distinctNodeBuildInfos = new HashMap<>(); AggregationFunctionCallEval[] aggFunctions = groupbyNode.getAggFunctions(); for (int aggIdx = 0; aggIdx < aggFunctions.length; aggIdx++) { AggregationFunctionCallEval aggFunction = aggFunctions[aggIdx]; @@ -223,7 +223,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou distinctNodeBuildInfos.put(groupbyMapKey, buildInfo); // Grouping columns are GROUP BY clause's column + Distinct column. - List groupingColumns = new ArrayList(); + List groupingColumns = new ArrayList<>(); for (Column eachGroupingColumn: groupbyUniqColumns) { if (!groupingColumns.contains(eachGroupingColumn)) { groupingColumns.add(eachGroupingColumn); @@ -239,7 +239,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou } } - List baseGroupByTargets = new ArrayList(); + List baseGroupByTargets = new ArrayList<>(); baseGroupByTargets.add(new Target(new FieldEval(new Column("?distinctseq", Type.INT2)))); for (Column column : originalGroupingColumns) { baseGroupByTargets.add(new Target(new FieldEval(column))); @@ -369,13 +369,13 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou */ List originalGroupingColumns = Arrays.asList(groupbyNode.getGroupingColumns()); - List childGroupbyNodes = new ArrayList(); + List childGroupbyNodes = new ArrayList<>(); - List otherAggregationFunctionCallEvals = new ArrayList(); - List otherAggregationFunctionTargets = new ArrayList(); + List otherAggregationFunctionCallEvals = new ArrayList<>(); + List otherAggregationFunctionTargets = new ArrayList<>(); //distinct columns -> GroupbyNode - Map distinctNodeBuildInfos = new HashMap(); + Map distinctNodeBuildInfos = new HashMap<>(); AggregationFunctionCallEval[] aggFunctions = groupbyNode.getAggFunctions(); for (int aggIdx = 0; aggIdx < aggFunctions.length; aggIdx++) { @@ -393,7 +393,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou distinctNodeBuildInfos.put(groupbyMapKey, buildInfo); // Grouping columns are GROUP BY clause's column + Distinct column. - List groupingColumns = new ArrayList(originalGroupingColumns); + List groupingColumns = new ArrayList<>(originalGroupingColumns); for (Column eachGroupingColumn: groupbyUniqColumns) { if (!groupingColumns.contains(eachGroupingColumn)) { groupingColumns.add(eachGroupingColumn); @@ -525,7 +525,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou // FirstStage: Remove aggregation, Set target with only grouping columns firstStageGroupbyNode.setAggFunctions(PlannerUtil.EMPTY_AGG_FUNCS); - List firstGroupbyTargets = new ArrayList(); + List firstGroupbyTargets = new ArrayList<>(); for (Column column : firstStageGroupbyNode.getGroupingColumns()) { Target target = new Target(new FieldEval(column)); firstGroupbyTargets.add(target); @@ -538,9 +538,9 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou secondStageGroupbyNode.setGroupingColumns(originGroupColumns.toArray(new Column[]{})); Target[] oldTargets = secondStageGroupbyNode.getTargets(); - List secondGroupbyTargets = new ArrayList(); + List secondGroupbyTargets = new ArrayList<>(); LinkedHashSet distinctColumns = EvalTreeUtil.findUniqueColumns(secondStageGroupbyNode.getAggFunctions()[0]); - List uniqueDistinctColumn = new ArrayList(); + List uniqueDistinctColumn = new ArrayList<>(); // remove origin group by column from distinctColumns for (Column eachColumn: distinctColumns) { if (!originGroupColumns.contains(eachColumn)) { @@ -570,7 +570,7 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou secondStageGroupbyNode.setTargets(secondGroupbyTargets.toArray(new Target[]{})); } else { // FirstStage: Change target of aggFunction to function name expr - List firstGroupbyTargets = new ArrayList(); + List firstGroupbyTargets = new ArrayList<>(); for (Column column : firstStageDistinctNode.getGroupingColumns()) { firstGroupbyTargets.add(new Target(new FieldEval(column))); columnIdIndex++; @@ -632,9 +632,9 @@ select col1, count(distinct col2), count(distinct col3), sum(col4) from ... grou } // Set FirstStage DistinctNode's target with grouping column and other aggregation function - List firstStageColumnIds = new ArrayList(); + List firstStageColumnIds = new ArrayList<>(); columnIdIndex = 0; - List firstTargets = new ArrayList(); + List firstTargets = new ArrayList<>(); for (GroupbyNode firstStageGroupbyNode: firstStageDistinctNode.getSubPlans()) { if (firstStageGroupbyNode.isDistinct()) { for (Column column : firstStageGroupbyNode.getGroupingColumns()) { @@ -684,10 +684,10 @@ private void setDistinctAggregationEnforcer( firstStageBlock.getEnforcer().enforceDistinctAggregation(firstStageDistinctNode.getPID(), DistinctAggregationAlgorithm.HASH_AGGREGATION, null); - List sortSpecArrays = new ArrayList(); + List sortSpecArrays = new ArrayList<>(); int index = 0; for (GroupbyNode groupbyNode: firstStageDistinctNode.getSubPlans()) { - List sortSpecs = new ArrayList(); + List sortSpecs = new ArrayList<>(); for (Column column: groupbyNode.getGroupingColumns()) { sortSpecs.add(SortSpecProto.newBuilder().setColumn(column.getProto()).build()); } @@ -712,10 +712,10 @@ private void setMultiStageAggregationEnforcer( true, MultipleAggregationStage.SECOND_STAGE, DistinctAggregationAlgorithm.HASH_AGGREGATION, null); - List sortSpecArrays = new ArrayList(); + List sortSpecArrays = new ArrayList<>(); int index = 0; for (GroupbyNode groupbyNode: firstStageDistinctNode.getSubPlans()) { - List sortSpecs = new ArrayList(); + List sortSpecs = new ArrayList<>(); for (Column column: groupbyNode.getGroupingColumns()) { sortSpecs.add(SortSpecProto.newBuilder().setColumn(column.getProto()).build()); } @@ -771,8 +771,8 @@ private ExecutionBlock buildDistinctGroupbyAndUnionPlan(MasterPlan masterPlan, E static class DistinctGroupbyNodeBuildInfo { private GroupbyNode groupbyNode; - private List aggFunctions = new ArrayList(); - private List aggFunctionTargets = new ArrayList(); + private List aggFunctions = new ArrayList<>(); + private List aggFunctionTargets = new ArrayList<>(); public DistinctGroupbyNodeBuildInfo(GroupbyNode groupbyNode) { this.groupbyNode = groupbyNode; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteEngine.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteEngine.java index f71c6837ba..7132e78b8a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/rewriter/GlobalPlanRewriteEngine.java @@ -33,7 +33,7 @@ public class GlobalPlanRewriteEngine { private static final Log LOG = LogFactory.getLog(GlobalPlanRewriteEngine.class); /** a map for query rewrite rules */ - private final Map rewriteRules = new LinkedHashMap(); + private final Map rewriteRules = new LinkedHashMap<>(); /** * Add a query rewrite rule to this engine. diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java index e1713387b0..19b0c9a199 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java @@ -133,7 +133,7 @@ public void init() throws IOException { // the target can be an empty list. if (plan.hasTargets()) { projected = new Schema(); - Set columnSet = new HashSet(); + Set columnSet = new HashSet<>(); if (plan.hasQual()) { columnSet.addAll(EvalTreeUtil.findUniqueColumns(qual)); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java index a248d52595..59239a6b2b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java @@ -158,7 +158,7 @@ protected TupleMap buildRightToHashTableForCrossJoin() throws IOExcep protected TupleMap buildRightToHashTableForNonCrossJoin() throws IOException { Tuple tuple; - TupleMap map = new TupleMap(100000); + TupleMap map = new TupleMap<>(100000); KeyProjector keyProjector = new KeyProjector(rightSchema, rightKeyList); while (!context.isStopped() && (tuple = rightChild.next()) != null) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java index 7ac3e0b413..92438fbbb3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java @@ -237,7 +237,7 @@ class NonDistinctHashAggregator { private final Tuple outTuple; private NonDistinctHashAggregator(GroupbyNode groupbyNode) throws IOException { - nonDistinctAggrDatas = new TupleMap(); + nonDistinctAggrDatas = new TupleMap<>(); if (groupbyNode.hasAggFunctions()) { aggFunctions = groupbyNode.getAggFunctions(); @@ -318,7 +318,7 @@ public DistinctHashAggregator(GroupbyNode groupbyNode, int nodeSequence) throws Set groupingKeySet = TUtil.newHashSet(plan.getGroupingColumns()); - List distinctGroupingKeyIndexSet = new ArrayList(); + List distinctGroupingKeyIndexSet = new ArrayList<>(); Column[] groupingColumns = groupbyNode.getGroupingColumns(); for (int idx = 0; idx < groupingColumns.length; idx++) { Column col = groupingColumns[idx]; @@ -330,7 +330,7 @@ public DistinctHashAggregator(GroupbyNode groupbyNode, int nodeSequence) throws distinctKeyColumns = distinctGroupingKeyIndexSet.toArray(distinctKeyColumns); this.dummyTuple = NullTuple.create(distinctGroupingKeyIndexSet.size()); - this.distinctAggrDatas = new TupleMap(); + this.distinctAggrDatas = new TupleMap<>(); distinctGroupbyKeyProjectors.put(nodeSequence, new KeyProjector(inSchema, distinctKeyColumns)); this.tupleLength = distinctKeyColumns.length; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java index 2d1fa4b0d3..418d0fc769 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java @@ -148,7 +148,7 @@ public Tuple next() throws IOException { // Groupby_Key2 | Distinct1_Column_V3 | | | //-------------------------------------------------------------------------------------- - List tupleSlots = new ArrayList(); + List tupleSlots = new ArrayList<>(); // aggregation with single grouping key for (int i = 0; i < hashAggregators.length; i++) { @@ -359,9 +359,9 @@ class HashAggregator { public HashAggregator(GroupbyNode groupbyNode, Schema schema) throws IOException { - hashTable = new TupleMap>(10000); + hashTable = new TupleMap<>(10000); - List groupingKeyColumnList = new ArrayList(distinctGroupingKeyColumnSet); + List groupingKeyColumnList = new ArrayList<>(distinctGroupingKeyColumnSet); Column[] keyColumns = groupbyNode.getGroupingColumns(); Column col; @@ -399,7 +399,7 @@ public void compute(Tuple tuple) throws IOException { TupleMap distinctEntry = hashTable.get(outerKeyTuple); if (distinctEntry == null) { - distinctEntry = new TupleMap(); + distinctEntry = new TupleMap<>(); hashTable.put(outerKeyTuple, distinctEntry); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySecondAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySecondAggregationExec.java index b3edab6ac9..be0b5132f2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySecondAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySecondAggregationExec.java @@ -86,8 +86,8 @@ public class DistinctGroupbySecondAggregationExec extends UnaryPhysicalExec { // Key tuples may have various lengths. The below two maps are used to cache key tuple instances. // Each map is a mapping of key length to key tuple. - private Map keyTupleMap = new HashMap(); - private Map prevKeyTupleMap = new HashMap(); + private Map keyTupleMap = new HashMap<>(); + private Map prevKeyTupleMap = new HashMap<>(); private Tuple prevKeyTuple = null; private Tuple prevTuple = null; @@ -109,7 +109,7 @@ public void init() throws IOException { List groupbyNodes = plan.getSubPlans(); // Finding distinct group by column index. - Set groupingKeyIndexSet = new HashSet(); + Set groupingKeyIndexSet = new HashSet<>(); for (Column col: plan.getGroupingColumns()) { int keyIndex; if (col.hasQualifier()) { @@ -140,7 +140,7 @@ public void init() throws IOException { distinctKeyIndexes = new int[numDistinct][]; for (GroupbyNode eachGroupby : groupbyNodes) { if (eachGroupby.isDistinct()) { - List distinctGroupingKeyIndex = new ArrayList(); + List distinctGroupingKeyIndex = new ArrayList<>(); Column[] distinctGroupingColumns = eachGroupby.getGroupingColumns(); for (int idx = 0; idx < distinctGroupingColumns.length; idx++) { Column col = distinctGroupingColumns[idx]; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java index 9e9e9b4201..7b3ca7d0e6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java @@ -72,7 +72,7 @@ public void init() throws IOException { List groupbyNodes = plan.getSubPlans(); - List aggregatorList = new ArrayList(); + List aggregatorList = new ArrayList<>(); int inTupleIndex = 1 + numGroupingColumns; int outTupleIndex = numGroupingColumns; int distinctSeq = 0; @@ -96,14 +96,14 @@ public void init() throws IOException { // make output schema mapping index resultTupleIndexes = new int[outSchema.size()]; - Map groupbyResultTupleIndex = new HashMap(); + Map groupbyResultTupleIndex = new HashMap<>(); int resultTupleIndex = 0; for (Column eachColumn: plan.getGroupingColumns()) { groupbyResultTupleIndex.put(eachColumn, resultTupleIndex); resultTupleIndex++; } for (GroupbyNode eachGroupby : groupbyNodes) { - Set groupingColumnSet = new HashSet(); + Set groupingColumnSet = new HashSet<>(); for (Column column: eachGroupby.getGroupingColumns()) { groupingColumnSet.add(column); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java index b657622dcd..e3f566a2b1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java @@ -42,7 +42,7 @@ public class HashAggregateExec extends AggregationExec { public HashAggregateExec(TaskAttemptContext ctx, GroupbyNode plan, PhysicalExec subOp) throws IOException { super(ctx, plan, subOp); hashKeyProjector = new KeyProjector(inSchema, plan.getGroupingColumns()); - hashTable = new TupleMap(10000); + hashTable = new TupleMap<>(10000); this.tuple = new VTuple(plan.getOutSchema().size()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java index 1860ec0a3d..c7987de240 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java @@ -40,7 +40,7 @@ public class HashBasedColPartitionStoreExec extends ColPartitionStoreExec { private final ComparableTuple partKey; - private final Map appenderMap = new HashMap(); + private final Map appenderMap = new HashMap<>(); public HashBasedColPartitionStoreExec(TaskAttemptContext context, StoreTableNode plan, PhysicalExec child) throws IOException { @@ -82,7 +82,7 @@ public Tuple next() throws IOException { getAppender(partKey, tuple).addTuple(tuple); } - List statSet = new ArrayList(); + List statSet = new ArrayList<>(); for (Appender app : appenderMap.values()) { app.flush(); app.close(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java index c463028090..7020b9d098 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java @@ -127,7 +127,7 @@ public Tuple next() throws IOException { @Override protected TupleMap> convert(TupleMap hashed, boolean fromCache) throws IOException { - TupleMap> tuples = new TupleMap>(hashed.size()); + TupleMap> tuples = new TupleMap<>(hashed.size()); for (Map.Entry entry : hashed.entrySet()) { // flag: initially false (whether this join key had at least one match on the counter part) tuples.putWihtoutKeyCopy(entry.getKey(), new Pair<>(false, entry.getValue())); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java index bc4382e051..a72a375251 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java @@ -45,7 +45,7 @@ public final class HashShuffleFileWriteExec extends UnaryPhysicalExec { private ShuffleFileWriteNode plan; private final TableMeta meta; private Partitioner partitioner; - private Map appenderMap = new HashMap(); + private Map appenderMap = new HashMap<>(); private final int numShuffleOutputs; private final int [] shuffleKeyIds; private HashShuffleAppenderManager hashShuffleAppenderManager; @@ -89,7 +89,7 @@ private HashShuffleAppender getAppender(int partId) throws IOException { return appender; } - Map partitionTuples = new HashMap(); + Map partitionTuples = new HashMap<>(); long writtenBytes = 0L; @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index e9f4fb93ab..2fc4fb417a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -84,13 +84,13 @@ public static CatalogProtos.FragmentProto[] getNonZeroLengthDataFiles(TajoConf t partitionDepth = tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns().size(); } - List nonZeroLengthFiles = new ArrayList(); + List nonZeroLengthFiles = new ArrayList<>(); if (fs.exists(path)) { getNonZeroLengthDataFiles(fs, path, nonZeroLengthFiles, fileIndex, numResultFiles, new AtomicInteger(0), tableDesc.hasPartition(), 0, partitionDepth); } - List fragments = new ArrayList(); + List fragments = new ArrayList<>(); String[] previousPartitionPathNames = null; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java index a41d20acfc..2572e1db8d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java @@ -143,7 +143,7 @@ public Schema getProjectSchema() { // the target can be an empty list. if (plan.hasTargets()) { projected = new Schema(); - Set columnSet = new HashSet(); + Set columnSet = new HashSet<>(); if (plan.hasQual()) { columnSet.addAll(EvalTreeUtil.findUniqueColumns(qual)); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index 7279e8ea6a..20f4b402c7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -44,7 +44,7 @@ public QueryContext(TajoConf conf) { public QueryContext(TajoConf conf, Session session) { super(conf, ConfigKey.ConfigType.QUERY, ConfigKey.ConfigType.SESSION); - Map copy = new HashMap(session.getAllVariables()); + Map copy = new HashMap<>(session.getAllVariables()); // Among session variables, timezone must not be copy.remove("TIMEZONE"); putAll(copy); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java index f97d00507f..7b52dabcb8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/TaskRequestImpl.java @@ -118,7 +118,7 @@ public List getFragments() { return fragments; } if (fragments == null) { - fragments = new ArrayList(); + fragments = new ArrayList<>(); } for (int i = 0; i < p.getFragmentsCount(); i++) { fragments.add(p.getFragments(i)); @@ -223,7 +223,7 @@ private void initFetches() { return; } TaskRequestProtoOrBuilder p = viaProto ? proto : builder; - this.fetches = new ArrayList(); + this.fetches = new ArrayList<>(); for(FetchProto fetch : p.getFetchesList()) { fetches.add(new FetchImpl(fetch)); } diff --git a/tajo-core/src/main/java/org/apache/tajo/ha/HdfsServiceTracker.java b/tajo-core/src/main/java/org/apache/tajo/ha/HdfsServiceTracker.java index 4b97fe6994..e7f4748b85 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ha/HdfsServiceTracker.java +++ b/tajo-core/src/main/java/org/apache/tajo/ha/HdfsServiceTracker.java @@ -596,7 +596,7 @@ public int formatHA(TajoConf conf) throws ServiceTrackerException{ @Override public List getMasters(TajoConf conf) throws ServiceTrackerException { - List list = new ArrayList(); + List list = new ArrayList<>(); try { FileSystem fs = getFileSystem(conf); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java index 8e999c335b..f76e7f0e1b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java @@ -103,7 +103,7 @@ public void kill() { try { getQueryInfo().setQueryState(TajoProtos.QueryState.QUERY_KILLED); if (queryMasterRpcClient != null) { - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); queryMasterRpcClient.killQuery(callFuture.getController(), queryId.getProto(), callFuture); callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); } @@ -157,7 +157,7 @@ protected boolean allocateToQueryMaster(AllocationResourceProto allocation) { connectQueryMaster(connectionInfo); } - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); queryMasterRpcClient.allocateQueryMaster(callFuture.getController(), allocation, callFuture); if(!callFuture.get().getValue()) return false; @@ -220,7 +220,7 @@ public boolean submitToQueryMaster() { .setLogicalPlanJson(PrimitiveProtos.StringProto.newBuilder().setValue(plan.toJson()).build()) .setAllocation(allocationResource); - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); queryMasterRpcClient.executeQuery(callFuture.getController(), builder.build(), callFuture); callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java index ba421bd01f..24ed830eb9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryManager.java @@ -153,7 +153,7 @@ public List getFinishedQueries(int page, int size) { result.addAll(historyCache.values()); } int fromIndex = (page - 1) * size; - return new LinkedList(result).subList(fromIndex, fromIndex + size); + return new LinkedList<>(result).subList(fromIndex, fromIndex + size); } else { try { return this.masterContext.getHistoryReader().getQueriesInHistory(page, size); @@ -164,7 +164,7 @@ public List getFinishedQueries(int page, int size) { synchronized (historyCache) { result.addAll(historyCache.values()); } - return new LinkedList(result); + return new LinkedList<>(result); } } } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index 227af0bd76..fa36b793a8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@ -393,7 +393,7 @@ public GetQueryListResponse getRunningQueryList(RpcController controller, TajoId try { context.getSessionManager().touch(request.getId()); - Collection queries = new ArrayList(context.getQueryJobManager().getSubmittedQueries()); + Collection queries = new ArrayList<>(context.getQueryJobManager().getSubmittedQueries()); queries.addAll(context.getQueryJobManager().getRunningQueries()); BriefQueryInfo.Builder infoBuilder = BriefQueryInfo.newBuilder(); @@ -666,7 +666,7 @@ public GetClusterInfoResponse getClusterInfo(RpcController controller, try { context.getSessionManager().touch(request.getSessionId().getId()); - List nodeStatusList = new ArrayList(context.getResourceManager().getRMContext().getNodes().values()); + List nodeStatusList = new ArrayList<>(context.getResourceManager().getRMContext().getNodes().values()); Collections.sort(nodeStatusList); WorkerResourceInfo.Builder workerBuilder = WorkerResourceInfo.newBuilder(); @@ -931,7 +931,7 @@ public FunctionListResponse getFunctionList(RpcController controller, SessionedS String functionName = request.getValue(); Collection functions = catalog.getFunctions(); - List functionProtos = new ArrayList(); + List functionProtos = new ArrayList<>(); for (FunctionDesc eachFunction: functions) { if (functionName == null || functionName.isEmpty()) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 15abf9e120..92e1775a6f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -331,7 +331,7 @@ public void truncateTable(final QueryContext queryContext, final TruncateTableNo String databaseName; String simpleTableName; - List tableDescList = new ArrayList(); + List tableDescList = new ArrayList<>(); for (String eachTableName : tableNames) { if (CatalogUtil.isFQTableName(eachTableName)) { String[] split = CatalogUtil.splitFQTableName(eachTableName); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainPlanPreprocessorForTest.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainPlanPreprocessorForTest.java index 0c83206cc8..b747849681 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainPlanPreprocessorForTest.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/ExplainPlanPreprocessorForTest.java @@ -50,7 +50,7 @@ public void prepareTest(LogicalPlan plan) throws TajoException { private static class PlanShapeFixerContext { - Stack childNumbers = new Stack(); + Stack childNumbers = new Stack<>(); public void reset() { childNumbers.clear(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index 14077b1624..82bc679caf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -157,7 +157,7 @@ public void close() { private List getTablespaces(Schema outSchema) { List tablespaces = masterContext.getCatalog().getAllTablespaces(); - List tuples = new ArrayList(tablespaces.size()); + List tuples = new ArrayList<>(tablespaces.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; @@ -192,7 +192,7 @@ private List getTablespaces(Schema outSchema) { private List getDatabases(Schema outSchema) { List databases = masterContext.getCatalog().getAllDatabases(); - List tuples = new ArrayList(databases.size()); + List tuples = new ArrayList<>(databases.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; @@ -222,7 +222,7 @@ private List getDatabases(Schema outSchema) { private List getTables(Schema outSchema) { List tables = masterContext.getCatalog().getAllTables(); - List tuples = new ArrayList(tables.size()); + List tuples = new ArrayList<>(tables.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; @@ -258,7 +258,7 @@ private List getTables(Schema outSchema) { private List getColumns(Schema outSchema) { List columnsList = masterContext.getCatalog().getAllColumns(); - List tuples = new ArrayList(columnsList.size()); + List tuples = new ArrayList<>(columnsList.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; int columnId = 1, prevtid = -1, tid = 0; @@ -306,7 +306,7 @@ private List getColumns(Schema outSchema) { private List getIndexes(Schema outSchema) { List indexList = masterContext.getCatalog().getAllIndexes(); - List tuples = new ArrayList(indexList.size()); + List tuples = new ArrayList<>(indexList.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; @@ -337,7 +337,7 @@ private List getIndexes(Schema outSchema) { private List getAllTableOptions(Schema outSchema) { List optionList = masterContext.getCatalog().getAllTableOptions(); - List tuples = new ArrayList(optionList.size()); + List tuples = new ArrayList<>(optionList.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; @@ -364,7 +364,7 @@ private List getAllTableOptions(Schema outSchema) { private List getAllTableStats(Schema outSchema) { List statList = masterContext.getCatalog().getAllTableStats(); - List tuples = new ArrayList(statList.size()); + List tuples = new ArrayList<>(statList.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; @@ -391,7 +391,7 @@ private List getAllTableStats(Schema outSchema) { private List getAllPartitions(Schema outSchema) { List partitionList = masterContext.getCatalog().getAllPartitions(); - List tuples = new ArrayList(partitionList.size()); + List tuples = new ArrayList<>(partitionList.size()); List columns = outSchema.getRootColumns(); Tuple aTuple; @@ -521,15 +521,15 @@ private Tuple getWorkerTuple(Schema outSchema, NodeStatus aNodeStatus) { private List getClusterInfo(Schema outSchema) { Map workerMap = masterContext.getResourceManager().getNodes(); List tuples; - List queryMasterList = new ArrayList(); - List nodeStatusList = new ArrayList(); + List queryMasterList = new ArrayList<>(); + List nodeStatusList = new ArrayList<>(); for (NodeStatus aNodeStatus : workerMap.values()) { queryMasterList.add(aNodeStatus); nodeStatusList.add(aNodeStatus); } - tuples = new ArrayList(queryMasterList.size() + nodeStatusList.size()); + tuples = new ArrayList<>(queryMasterList.size() + nodeStatusList.size()); for (NodeStatus queryMaster: queryMasterList) { tuples.add(getQueryMasterTuple(outSchema, queryMaster)); } @@ -591,7 +591,7 @@ private List fetchSystemTable(TableDesc tableDesc, Schema inSchema) { @Override public List getNextRows(int fetchRowNum) throws IOException { - List rows = new ArrayList(); + List rows = new ArrayList<>(); int startRow = currentRow; int endRow = startRow + fetchRowNum; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index d5341d80db..6219e3198d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -510,7 +510,7 @@ private void insertRowValues(QueryContext queryContext, } else { // If INSERT INTO LOCATION // Empty TableDesc - List columns = new ArrayList(); + List columns = new ArrayList<>(); CatalogProtos.TableDescProto tableDescProto = CatalogProtos.TableDescProto.newBuilder() .setTableName(nodeUniqName) .setMeta(CatalogProtos.TableProto.newBuilder().setStoreType(BuiltinStorages.TEXT).build()) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/prehook/DistributedQueryHookManager.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/prehook/DistributedQueryHookManager.java index f403092f34..87ebd9dbb1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/prehook/DistributedQueryHookManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/prehook/DistributedQueryHookManager.java @@ -25,7 +25,7 @@ import java.util.List; public class DistributedQueryHookManager { - private List hooks = new ArrayList(); + private List hooks = new ArrayList<>(); public void addHook(DistributedQueryHook hook) { hooks.add(hook); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rule/MasterRuleProvider.java b/tajo-core/src/main/java/org/apache/tajo/master/rule/MasterRuleProvider.java index 0caedb99cf..1a609cf5b5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/rule/MasterRuleProvider.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/rule/MasterRuleProvider.java @@ -37,7 +37,7 @@ public class MasterRuleProvider implements SelfDiagnosisRuleProvider { public List getDefinedRules() { Set classSet = ClassUtil.findClasses(SelfDiagnosisRule.class, getClass().getPackage().getName()); - List ruleList = new ArrayList(classSet.size()); + List ruleList = new ArrayList<>(classSet.size()); for (Class ruleClazz: classSet) { try { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SimpleScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SimpleScheduler.java index e41ac95980..699461f5f4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SimpleScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/scheduler/SimpleScheduler.java @@ -75,7 +75,7 @@ public SimpleScheduler(TajoMaster.MasterContext context, TajoRMContext rmContext this.masterContext = context; this.rmContext = rmContext; //Copy default array capacity from PriorityBlockingQueue. - this.queryQueue = new PriorityBlockingQueue(11, COMPARATOR); + this.queryQueue = new PriorityBlockingQueue<>(11, COMPARATOR); this.queryProcessor = new Thread(new QueryProcessor()); } @@ -184,7 +184,7 @@ public int getNumClusterNodes() { return Lists.newArrayList(); } - LinkedList workers = new LinkedList(); + LinkedList workers = new LinkedList<>(); if (request.getCandidateNodesCount() > 0) { workers.addAll(request.getCandidateNodesList()); @@ -197,7 +197,7 @@ public int getNumClusterNodes() { // reserve resource in random workers if (reservedResources.size() < requiredContainers) { - LinkedList randomNodes = new LinkedList(getRMContext().getNodes().keySet()); + LinkedList randomNodes = new LinkedList<>(getRMContext().getNodes().keySet()); Collections.shuffle(randomNodes); reservedResources.addAll(reserveClusterResource( diff --git a/tajo-core/src/main/java/org/apache/tajo/metrics/ClusterResourceMetricSet.java b/tajo-core/src/main/java/org/apache/tajo/metrics/ClusterResourceMetricSet.java index 7454d21c5b..1798f22e88 100644 --- a/tajo-core/src/main/java/org/apache/tajo/metrics/ClusterResourceMetricSet.java +++ b/tajo-core/src/main/java/org/apache/tajo/metrics/ClusterResourceMetricSet.java @@ -37,7 +37,7 @@ public ClusterResourceMetricSet(TajoMaster.MasterContext masterContext) { @Override public Map getMetrics() { - Map metricsMap = new HashMap(); + Map metricsMap = new HashMap<>(); metricsMap.put(Cluster.TOTAL_NODES.name(), new Gauge() { @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java index 6190cdc348..4526144870 100644 --- a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java @@ -345,7 +345,7 @@ public Aggregation visitGroupby_clause(Groupby_clauseContext ctx) { // If grouping group is not empty if (ctx.grouping_element_list().grouping_element().get(0).empty_grouping_set() == null) { int elementSize = ctx.grouping_element_list().grouping_element().size(); - ArrayList groups = new ArrayList(elementSize + 1); + ArrayList groups = new ArrayList<>(elementSize + 1); ArrayList ordinaryExprs = null; int groupSize = 1; groups.add(null); @@ -355,7 +355,7 @@ public Aggregation visitGroupby_clause(Groupby_clauseContext ctx) { ctx.grouping_element_list().grouping_element().get(i); if (element.ordinary_grouping_set() != null) { if (ordinaryExprs == null) { - ordinaryExprs = new ArrayList(); + ordinaryExprs = new ArrayList<>(); } Collections.addAll(ordinaryExprs, getRowValuePredicandsFromOrdinaryGroupingSet(element.ordinary_grouping_set())); } else if (element.rollup_list() != null) { @@ -636,7 +636,7 @@ public Join visitJoined_table_primary(Joined_table_primaryContext ctx) { } private Expr[] getRowValuePredicandsFromOrdinaryGroupingSetList(Ordinary_grouping_set_listContext ctx) { - ArrayList rowValuePredicands = new ArrayList(); + ArrayList rowValuePredicands = new ArrayList<>(); for (int i = 0; i < ctx.ordinary_grouping_set().size(); i++) { Collections.addAll(rowValuePredicands, getRowValuePredicandsFromOrdinaryGroupingSet(ctx.ordinary_grouping_set(i))); } @@ -644,7 +644,7 @@ private Expr[] getRowValuePredicandsFromOrdinaryGroupingSetList(Ordinary_groupin } private Expr[] getRowValuePredicandsFromOrdinaryGroupingSet(Ordinary_grouping_setContext ctx) { - ArrayList rowValuePredicands = new ArrayList(); + ArrayList rowValuePredicands = new ArrayList<>(); if (ctx.row_value_predicand() != null) { rowValuePredicands.add(visitRow_value_predicand(ctx.row_value_predicand())); } @@ -1357,7 +1357,7 @@ public Expr visitCreate_table_statement(Create_table_statementContext ctx) { @Override public Expr visitTruncate_table_statement(@NotNull Truncate_table_statementContext ctx) { List tableNameContexts = ctx.table_name(); - List tableNames = new ArrayList(); + List tableNames = new ArrayList<>(); for (Table_nameContext eachTableNameContext: tableNameContexts) { tableNames.add(buildIdentifierChain(eachTableNameContext.identifier())); @@ -1688,7 +1688,7 @@ public Expr visitDrop_table_statement(Drop_table_statementContext ctx) { private Map getParams(Param_clauseContext ctx) { - Map params = new HashMap(); + Map params = new HashMap<>(); for (int i = 0; i < ctx.param().size(); i++) { params.put(stripQuote(ctx.param(i).key.getText()), stripQuote(ctx.param(i).value.getText())); } @@ -1697,7 +1697,7 @@ private Map getParams(Param_clauseContext ctx) { } public Map escapeTableMeta(Map map) { - Map params = new HashMap(); + Map params = new HashMap<>(); for (Map.Entry entry : map.entrySet()) { if (entry.getKey().equals(StorageConstants.TEXT_DELIMITER)) { params.put(StorageConstants.TEXT_DELIMITER, StringUtils.unicodeEscapedDelimiter(entry.getValue())); @@ -1929,7 +1929,7 @@ public Expr visitAlter_table_statement(Alter_table_statementContext ctx) { } private Map getProperties(Property_listContext ctx) { - Map params = new HashMap(); + Map params = new HashMap<>(); for (int i = 0; i < ctx.property().size(); i++) { params.put(stripQuote(ctx.property(i).key.getText()), stripQuote(ctx.property(i).value.getText())); } diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java index 25695a599a..c705084b78 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/DefaultTaskScheduler.java @@ -286,7 +286,7 @@ private Set getWorkerIds(Collection hosts){ protected LinkedList createTaskRequest(final int incompleteTaskNum) throws Exception { - LinkedList taskRequestEvents = new LinkedList(); + LinkedList taskRequestEvents = new LinkedList<>(); //If scheduled tasks is long-term task, cluster resource can be the worst load balance. //This part is to throttle the maximum required container per request @@ -301,7 +301,7 @@ protected LinkedList createTaskRequest(final int incompleteTas getClient(serviceTracker.getUmbilicalAddress(), QueryCoordinatorProtocol.class, true, rpcParams); QueryCoordinatorProtocolService masterClientService = tmClient.getStub(); - CallFuture callBack = new CallFuture(); + CallFuture callBack = new CallFuture<>(); NodeResourceRequest.Builder request = NodeResourceRequest.newBuilder(); request.setCapacity(NodeResources.createResource(minTaskMemory, isLeaf ? 1 : 0).getProto()) .setNumContainers(requestContainerNum) @@ -380,7 +380,7 @@ public class HostVolumeMapping { * These disk volumes are kept in an order of ascending order of the volume id. * In other words, the head volume ids are likely to -1, meaning no given volume id. */ - private SortedMap diskVolumeLoads = new TreeMap(); + private SortedMap diskVolumeLoads = new TreeMap<>(); /** The total number of remain tasks in this host */ private AtomicInteger remainTasksNum = new AtomicInteger(0); public static final int REMOTE = -2; @@ -395,7 +395,7 @@ public synchronized void addTaskAttempt(int volumeId, TaskAttempt attemptId){ synchronized (unassignedTaskForEachVolume){ LinkedHashSet list = unassignedTaskForEachVolume.get(volumeId); if (list == null) { - list = new LinkedHashSet(); + list = new LinkedHashSet<>(); unassignedTaskForEachVolume.put(volumeId, list); } list.add(attemptId); @@ -653,7 +653,7 @@ private void addLeafTask(TaskAttemptToSchedulerEvent event) { HashSet list = leafTasksRackMapping.get(hostVolumeMapping.getRack()); if (list == null) { - list = new HashSet(); + list = new HashSet<>(); leafTasksRackMapping.put(hostVolumeMapping.getRack(), list); } @@ -761,7 +761,7 @@ public int compare(HostVolumeMapping v1, HostVolumeMapping v2) { public void assignToLeafTasks(LinkedList taskRequests) { Collections.shuffle(taskRequests); - LinkedList remoteTaskRequests = new LinkedList(); + LinkedList remoteTaskRequests = new LinkedList<>(); String queryMasterHostAndPort = context.getMasterContext().getQueryMasterContext().getWorkerContext(). getConnectionInfo().getHostAndQMPort(); @@ -862,7 +862,7 @@ public void assignToLeafTasks(LinkedList taskRequests) { Task task = stage.getTask(attemptId.getTaskId()); TaskRequest taskAssign = new TaskRequestImpl( attemptId, - new ArrayList(task.getAllFragments()), + new ArrayList<>(task.getAllFragments()), "", false, LogicalNodeSerializer.serialize(task.getLogicalPlan()), @@ -887,7 +887,7 @@ public void assignToLeafTasks(LinkedList taskRequests) { if (addr == null) addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort()); AsyncRpcClient tajoWorkerRpc = null; - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); totalAttempts++; try { tajoWorkerRpc = RpcClientManager.getInstance().getClient(addr, TajoWorkerProtocol.class, true, @@ -1003,7 +1003,7 @@ public void assignToNonLeafTasks(LinkedList taskRequests) { requestProto.setExecutionBlockId(attemptId.getTaskId().getExecutionBlockId().getProto()); context.getMasterContext().getEventHandler().handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo)); - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); InetSocketAddress addr = stage.getAssignedWorkerMap().get(connectionInfo.getId()); if (addr == null) addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort()); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index f06d28c20c..10c7ffee8c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -86,7 +86,7 @@ public class Query implements EventHandler { private int killedStagesCount = 0; private int failedStagesCount = 0; private int erroredStagesCount = 0; - private final List diagnostics = new ArrayList(); + private final List diagnostics = new ArrayList<>(); // Internal Variables private final Lock readLock; @@ -248,7 +248,7 @@ public float getProgress() { return 1.0f; } else { int idx = 0; - List tempStages = new ArrayList(); + List tempStages = new ArrayList<>(); synchronized(stages) { tempStages.addAll(stages.values()); } @@ -301,7 +301,7 @@ public QueryHistory getQueryHistory() { } private List makeStageHistories() { - List stageHistories = new ArrayList(); + List stageHistories = new ArrayList<>(); for(Stage eachStage : getStages()) { stageHistories.add(eachStage.getStageHistory()); } @@ -318,7 +318,7 @@ private QueryHistory makeQueryHistory() { queryHistory.setLogicalPlan(plan.getLogicalPlan().toString()); queryHistory.setDistributedPlan(plan.toString()); - List sessionVariables = new ArrayList(); + List sessionVariables = new ArrayList<>(); for(Map.Entry entry: plan.getContext().getAllKeyValus().entrySet()) { if (SessionVars.exists(entry.getKey()) && SessionVars.isPublic(SessionVars.get(entry.getKey()))) { sessionVariables.add(new String[]{entry.getKey(), entry.getValue()}); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java index 1b900809ae..6b26ddf1bb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java @@ -176,7 +176,7 @@ public List getAllWorker() { rpcClientParams); QueryCoordinatorProtocolService masterService = rpc.getStub(); - CallFuture callBack = new CallFuture(); + CallFuture callBack = new CallFuture<>(); masterService.getAllWorkers(callBack.getController(), PrimitiveProtos.NullProto.getDefaultInstance(), callBack); @@ -186,7 +186,7 @@ public List getAllWorker() { } catch (Exception e) { LOG.error(e.getMessage(), e); } - return new ArrayList(); + return new ArrayList<>(); } @Override @@ -291,7 +291,7 @@ public void stopQuery(final QueryId queryId) { queryMasterTasks.remove(queryId); TajoHeartbeatRequest queryHeartbeat = buildTajoHeartBeat(queryMasterTask); - CallFuture future = new CallFuture(); + CallFuture future = new CallFuture<>(); NettyClientBase tmClient; try { @@ -397,7 +397,7 @@ public QueryHeartbeatThread() { public void run() { LOG.info("Start QueryMaster heartbeat thread"); while(!isStopped) { - List tempTasks = new ArrayList(); + List tempTasks = new ArrayList<>(); tempTasks.addAll(queryMasterTasks.values()); for(QueryMasterTask eachTask: tempTasks) { @@ -439,7 +439,7 @@ public void run() { } catch (InterruptedException e) { break; } - List tempTasks = new ArrayList(); + List tempTasks = new ArrayList<>(); tempTasks.addAll(queryMasterTasks.values()); for(QueryMasterTask eachTask: tempTasks) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java index 05e7165e33..c911fbcc9c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java @@ -245,7 +245,7 @@ public void executeQuery(RpcController controller, public void allocateQueryMaster(RpcController controller, AllocationResourceProto request, RpcCallback done) { - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); workerContext.getNodeResourceManager().handle(new QMResourceAllocateEvent(request, callFuture)); try { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index bcfb9388b1..d6744c3bb1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@ -106,7 +106,7 @@ public class QueryMasterTask extends CompositeService { private NodeResource allocation; - private final List diagnostics = new ArrayList(); + private final List diagnostics = new ArrayList<>(); private final ConcurrentMap workerMap = Maps.newConcurrentMap(); @@ -261,7 +261,7 @@ protected void killTaskAttempt(int workerId, TaskAttemptId taskAttemptId) { tajoWorkerRpc = RpcClientManager.getInstance().getClient(workerAddress, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), callFuture); if(!callFuture.get().getValue()){ diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index bddf198a84..9e05882bf8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -187,8 +187,8 @@ public static void scheduleFragmentsForJoinQuery(TaskSchedulerContext schedulerC // Assigning either fragments or fetch urls to query units if (execBlock.hasBroadcastRelation()) { // If some relations of this EB are broadcasted boolean hasNonLeafNode = false; - List largeScanIndexList = new ArrayList(); - List broadcastIndexList = new ArrayList(); + List largeScanIndexList = new ArrayList<>(); + List broadcastIndexList = new ArrayList<>(); String nonLeafScanNames = ""; String namePrefix = ""; long maxStats = Long.MIN_VALUE; @@ -286,7 +286,7 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster // The hash map is modeling as follows: // >> Map>> hashEntries = - new HashMap>>(); + new HashMap<>(); // Grouping IntermediateData by a partition key and a table name List childBlocks = masterPlan.getChilds(stage.getId()); @@ -315,7 +315,7 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster } } else { Map> tbNameToInterm = - new HashMap>(); + new HashMap<>(); tbNameToInterm.put(scanEbId, TUtil.newList(intermediateEntry)); hashEntries.put(intermediateEntry.getPartId(), tbNameToInterm); } @@ -331,7 +331,7 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster tbNameToInterm.put(scanEbId, new ArrayList()); } else { Map> tbNameToInterm = - new HashMap>(); + new HashMap<>(); tbNameToInterm.put(scanEbId, new ArrayList()); hashEntries.put(emptyPartitionId, tbNameToInterm); } @@ -365,7 +365,7 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster int joinTaskNum = Math.min(maxTaskNum, hashEntries.size()); LOG.info("The determined number of join tasks is " + joinTaskNum); - List rightFragments = new ArrayList(); + List rightFragments = new ArrayList<>(); if (fragments.length == 2) { rightFragments.add(fragments[1]); } @@ -419,7 +419,7 @@ private static void scheduleSymmetricRepartitionJoin(QueryMasterTask.QueryMaster public static Map>> mergeIntermediateByPullHost( Map>> hashEntries) { Map>> mergedHashEntries = - new HashMap>>(); + new HashMap<>(); for(Entry>> entry: hashEntries.entrySet()) { Integer partId = entry.getKey(); @@ -432,7 +432,7 @@ public static Map>> merge // EBID + PullHost -> IntermediateEntry // In the case of union partEntry.getKey() return's delegated EBID. // Intermediate entries are merged by real EBID. - Map ebMerged = new HashMap(); + Map ebMerged = new HashMap<>(); for (IntermediateEntry eachIntermediate: intermediateList) { String ebMergedKey = eachIntermediate.getEbId().toString() + eachIntermediate.getPullHost().getPullAddress(); @@ -445,11 +445,11 @@ public static Map>> merge intermediateEntryPerPullHost.setVolume(intermediateEntryPerPullHost.getVolume() + eachIntermediate.getVolume()); } - List ebIntermediateEntries = new ArrayList(ebMerged.values()); + List ebIntermediateEntries = new ArrayList<>(ebMerged.values()); Map> mergedPartEntries = mergedHashEntries.get(partId); if (mergedPartEntries == null) { - mergedPartEntries = new HashMap>(); + mergedPartEntries = new HashMap<>(); mergedHashEntries.put(partId, mergedPartEntries); } mergedPartEntries.put(ebId, ebIntermediateEntries); @@ -498,7 +498,7 @@ private static void scheduleLeafTasksWithBroadcastTable(TaskSchedulerContext sch // -> SCAN // . add all fragments to broadcastFragments Collection baseFragments = null; - List broadcastFragments = new ArrayList(); + List broadcastFragments = new ArrayList<>(); for (int i = 0; i < scans.length; i++) { ScanNode scan = scans[i]; TableDesc desc = stage.getContext().getTableDesc(scan); @@ -543,7 +543,7 @@ private static void scheduleLeafTasksWithBroadcastTable(TaskSchedulerContext sch private static void addJoinShuffle(Stage stage, int partitionId, Map> grouppedPartitions) { - Map> fetches = new HashMap>(); + Map> fetches = new HashMap<>(); for (ExecutionBlock execBlock : stage.getMasterPlan().getChilds(stage.getId())) { if (grouppedPartitions.containsKey(execBlock.getId())) { Collection requests = mergeShuffleRequest(partitionId, HASH_SHUFFLE, @@ -569,7 +569,7 @@ private static Collection mergeShuffleRequest(int partitionId, ShuffleType type, List partitions) { // ebId + pullhost -> FetchImmpl - Map mergedPartitions = new HashMap(); + Map mergedPartitions = new HashMap<>(); for (IntermediateEntry partition : partitions) { String mergedKey = partition.getEbId().toString() + "," + partition.getPullHost(); @@ -603,7 +603,7 @@ public static void scheduleFragmentsForNonLeafTasks(TaskSchedulerContext schedul private static TableStats computeChildBlocksStats(QueryMasterTask.QueryMasterTaskContext context, MasterPlan masterPlan, ExecutionBlockId parentBlockId) { - List tableStatses = new ArrayList(); + List tableStatses = new ArrayList<>(); List childBlocks = masterPlan.getChilds(parentBlockId); for (ExecutionBlock childBlock : childBlocks) { Stage childStage = context.getStage(childBlock.getId()); @@ -696,7 +696,7 @@ public static void scheduleRangeShuffledFetches(TaskSchedulerContext schedulerCo new String[]{UNKNOWN_HOST}); Stage.scheduleFragment(stage, dummyFragment); - List fetches = new ArrayList(); + List fetches = new ArrayList<>(); List childBlocks = masterPlan.getChilds(stage.getId()); for (ExecutionBlock childBlock : childBlocks) { Stage childExecSM = stage.getContext().getStage(childBlock.getId()); @@ -710,13 +710,13 @@ public static void scheduleRangeShuffledFetches(TaskSchedulerContext schedulerCo } SortedMap> map; - map = new TreeMap>(); + map = new TreeMap<>(); Set fetchSet; try { RowStoreUtil.RowStoreEncoder encoder = RowStoreUtil.createEncoder(sortSchema); for (int i = 0; i < ranges.length; i++) { - fetchSet = new HashSet(); + fetchSet = new HashSet<>(); for (FetchImpl fetch: fetches) { String rangeParam = TupleUtil.rangeToQuery(ranges[i], i == (ranges.length - 1) , encoder); @@ -746,7 +746,7 @@ public static void scheduleFetchesByRoundRobin(Stage stage, Map>[] fetchesArray = new Map[num]; for (i = 0; i < num; i++) { - fetchesArray[i] = new HashMap>(); + fetchesArray[i] = new HashMap<>(); } i = 0; for (Entry> entry : partitions.entrySet()) { @@ -792,16 +792,15 @@ public static void scheduleHashShuffledFetches(TaskSchedulerContext schedulerCon // TODO - We should remove dummy fragment usages Fragment frag = new FileFragment(scan.getCanonicalName(), new Path("/dummy"), 0, 0, new String[]{UNKNOWN_HOST}); - List fragments = new ArrayList(); + List fragments = new ArrayList<>(); fragments.add(frag); Stage.scheduleFragments(stage, fragments); - Map finalFetches = new HashMap(); - Map> intermediates = new HashMap>(); + Map finalFetches = new HashMap<>(); + Map> intermediates = new HashMap<>(); for (ExecutionBlock block : masterPlan.getChilds(execBlock)) { - List partitions = new ArrayList(); + List partitions = new ArrayList<>(); partitions.addAll(stage.getContext().getStage(block.getId()).getHashShuffleIntermediateEntries()); // In scattered hash shuffle, Collecting each IntermediateEntry @@ -906,7 +905,7 @@ public int compare(FetchGroupMeta o1, FetchGroupMeta o2) { Long [] assignedVolumes = new Long[num]; // initialization for (int i = 0; i < num; i++) { - fetchesArray[i] = new HashMap>(); + fetchesArray[i] = new HashMap<>(); assignedVolumes[i] = 0l; } @@ -944,7 +943,7 @@ public int compare(FetchGroupMeta o1, FetchGroupMeta o2) { } } - return new Pair>[]>(assignedVolumes, fetchesArray); + return new Pair<>(assignedVolumes, fetchesArray); } public static void scheduleFetchesByEvenDistributedVolumes(Stage stage, Map partitions, @@ -974,12 +973,12 @@ public static void scheduleScatteredHashShuffleFetches(TaskSchedulerContext sche throw new RuntimeException("tajo.dist-query.table-partition.task-volume-mb should be great than " + "tajo.shuffle.hash.appender.page.volumn-mb"); } - List> fetches = new ArrayList>(); + List> fetches = new ArrayList<>(); long totalIntermediateSize = 0L; for (Entry> listEntry : intermediates.entrySet()) { // merge by PartitionId - Map> partitionIntermMap = new HashMap>(); + Map> partitionIntermMap = new HashMap<>(); for (IntermediateEntry eachInterm: listEntry.getValue()) { totalIntermediateSize += eachInterm.getVolume(); int partId = eachInterm.getPartId(); @@ -1007,7 +1006,7 @@ public static void scheduleScatteredHashShuffleFetches(TaskSchedulerContext sche int i = 0; Map>[] fetchesArray = new Map[fetches.size()]; for(List entry : fetches) { - fetchesArray[i] = new HashMap>(); + fetchesArray[i] = new HashMap<>(); fetchesArray[i].put(tableName, entry); Stage.scheduleFetches(stage, fetchesArray[i]); @@ -1031,13 +1030,13 @@ public static void scheduleScatteredHashShuffleFetches(TaskSchedulerContext sche public static List> splitOrMergeIntermediates( ExecutionBlockId ebId, List entries, long splitVolume, long pageSize) { // Each List has splitVolume size. - List> fetches = new ArrayList>(); + List> fetches = new ArrayList<>(); Iterator iter = entries.iterator(); if (!iter.hasNext()) { return null; } - List fetchListForSingleTask = new ArrayList(); + List fetchListForSingleTask = new ArrayList<>(); long fetchListVolume = 0; while (iter.hasNext()) { @@ -1060,7 +1059,7 @@ public static List> splitOrMergeIntermediates( if (!fetchListForSingleTask.isEmpty()) { fetches.add(fetchListForSingleTask); } - fetchListForSingleTask = new ArrayList(); + fetchListForSingleTask = new ArrayList<>(); fetchListVolume = 0; } FetchImpl fetch = new FetchImpl(currentInterm.getPullHost(), SCATTERED_HASH_SHUFFLE, @@ -1098,7 +1097,7 @@ public static List createFetchURL(FetchImpl fetch, boolean includeParts) { urlPrefix.append("&offset=").append(fetch.getOffset()).append("&length=").append(fetch.getLength()); } - List fetchURLs = new ArrayList(); + List fetchURLs = new ArrayList<>(); if(includeParts) { if (fetch.getType() == HASH_SHUFFLE || fetch.getType() == SCATTERED_HASH_SHUFFLE) { fetchURLs.add(URI.create(urlPrefix.toString())); @@ -1107,7 +1106,7 @@ public static List createFetchURL(FetchImpl fetch, boolean includeParts) { // the long request uri may cause HTTP Status Code - 414 Request-URI Too Long. // Refer to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.15 // The below code transforms a long request to multiple requests. - List taskIdsParams = new ArrayList(); + List taskIdsParams = new ArrayList<>(); StringBuilder taskIdListBuilder = new StringBuilder(); List taskIds = fetch.getTaskIds(); List attemptIds = fetch.getAttemptIds(); @@ -1156,7 +1155,7 @@ public static List createFetchURL(FetchImpl fetch, boolean includeParts) { } public static Map> hashByKey(List entries) { - Map> hashed = new HashMap>(); + Map> hashed = new HashMap<>(); for (IntermediateEntry entry : entries) { if (hashed.containsKey(entry.getPartId())) { hashed.get(entry.getPartId()).add(entry); @@ -1169,7 +1168,7 @@ public static Map> hashByKey(List> hashByHost(List entries) { - Map> hashed = new HashMap>(); + Map> hashed = new HashMap<>(); Task.PullHost host; for (IntermediateEntry entry : entries) { diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 98ad2920be..74edc480f9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -101,7 +101,7 @@ public class Stage implements EventHandler { private EventHandler eventHandler; private AbstractTaskScheduler taskScheduler; private QueryMasterTask.QueryMasterTaskContext context; - private final List diagnostics = new ArrayList(); + private final List diagnostics = new ArrayList<>(); private StageState stageState; private long startTime; @@ -374,7 +374,7 @@ public float getProgress() { if (getState() == StageState.NEW) { return 0.0f; } else { - tempTasks = new ArrayList(tasks.values()); + tempTasks = new ArrayList<>(tasks.values()); } } finally { readLock.unlock(); @@ -435,7 +435,7 @@ public StageHistory getStageHistory() { } private List makeTaskHistories() { - List taskHistories = new ArrayList(); + List taskHistories = new ArrayList<>(); for(Task eachTask : getTasks()) { taskHistories.add(eachTask.getTaskHistory()); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java index a586e4b377..7a9c8b1826 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Task.java @@ -191,10 +191,10 @@ public Task(Configuration conf, TaskAttemptScheduleContext scheduleContext, this.taskId = id; this.eventHandler = eventHandler; this.isLeafTask = isLeafTask; - scan = new ArrayList(); + scan = new ArrayList<>(); fetchMap = Maps.newHashMap(); fragMap = Maps.newHashMap(); - shuffleFileOutputs = new ArrayList(); + shuffleFileOutputs = new ArrayList<>(); attempts = Collections.emptyMap(); lastAttemptId = null; nextAttempt = -1; @@ -269,7 +269,7 @@ private TaskHistory makeTaskHistory() { } } - List fragmentList = new ArrayList(); + List fragmentList = new ArrayList<>(); for (FragmentProto eachFragment : getAllFragments()) { try { Fragment fragment = FragmentConvertor.convert(systemConf, eachFragment); @@ -281,7 +281,7 @@ private TaskHistory makeTaskHistory() { } taskHistory.setFragments(fragmentList.toArray(new String[]{})); - List fetchList = new ArrayList(); + List fetchList = new ArrayList<>(); for (Map.Entry> e : getFetchMap().entrySet()) { for (FetchImpl f : e.getValue()) { for (URI uri : f.getSimpleURIs()){ @@ -292,7 +292,7 @@ private TaskHistory makeTaskHistory() { taskHistory.setFetchs(fetchList.toArray(new String[][]{})); - List dataLocationList = new ArrayList(); + List dataLocationList = new ArrayList<>(); for(DataLocation eachLocation: getDataLocations()) { dataLocationList.add(eachLocation.toString()); } @@ -305,7 +305,7 @@ public void setLogicalPlan(LogicalNode plan) { this.plan = plan; LogicalNode node = plan; - ArrayList s = new ArrayList(); + ArrayList s = new ArrayList<>(); s.add(node); while (!s.isEmpty()) { node = s.remove(s.size()-1); @@ -340,7 +340,7 @@ public void addFragment(Fragment fragment, boolean useDataLocation) { if (fragMap.containsKey(fragment.getTableName())) { fragmentProtos = fragMap.get(fragment.getTableName()); } else { - fragmentProtos = new HashSet(); + fragmentProtos = new HashSet<>(); fragMap.put(fragment.getTableName(), fragmentProtos); } fragmentProtos.add(fragment.getProto()); @@ -381,7 +381,7 @@ public void setFetches(Map> fetches) { } public Collection getAllFragments() { - Set fragmentProtos = new HashSet(); + Set fragmentProtos = new HashSet<>(); for (Set eachFragmentSet : fragMap.values()) { fragmentProtos.addAll(eachFragmentSet); } @@ -547,7 +547,7 @@ private void addAndScheduleAttempt() { case 1: Map newAttempts - = new LinkedHashMap(3); + = new LinkedHashMap<>(3); newAttempts.putAll(attempts); attempts = newAttempts; attempts.put(attempt.getId(), attempt); @@ -716,7 +716,7 @@ public void handle(TaskEvent event) { } public void setIntermediateData(Collection partitions) { - this.intermediateData = new ArrayList(partitions); + this.intermediateData = new ArrayList<>(partitions); } public List getIntermediateData() { @@ -795,14 +795,14 @@ public IntermediateEntry(IntermediateEntryProto proto) { this.host = new PullHost(pullHost[0], Integer.parseInt(pullHost[1])); this.volume = proto.getVolume(); - failureRowNums = new ArrayList>>(); + failureRowNums = new ArrayList<>(); for (FailureIntermediateProto eachFailure: proto.getFailuresList()) { failureRowNums.add(new Pair(eachFailure.getPagePos(), new Pair(eachFailure.getStartRowNum(), eachFailure.getEndRowNum()))); } - pages = new ArrayList>(); + pages = new ArrayList<>(); for (IntermediateEntryProto.PageProto eachPage: proto.getPagesList()) { pages.add(new Pair(eachPage.getPos(), eachPage.getLength())); } @@ -873,7 +873,7 @@ public int hashCode() { } public List> split(long firstSplitVolume, long splitVolume) { - List> splits = new ArrayList>(); + List> splits = new ArrayList<>(); if (pages == null || pages.isEmpty()) { return splits; diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/TaskAttempt.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/TaskAttempt.java index cda62a4154..3f03411ac9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/TaskAttempt.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/TaskAttempt.java @@ -59,7 +59,7 @@ public class TaskAttempt implements EventHandler { private final Lock readLock; private final Lock writeLock; - private final List diagnostics = new ArrayList(); + private final List diagnostics = new ArrayList<>(); private final TaskAttemptScheduleContext scheduleContext; @@ -266,7 +266,7 @@ public void addPartitions(List partitions) { private void fillTaskStatistics(TaskCompletionReport report) { this.progress = 1.0f; - List partitions = new ArrayList(); + List partitions = new ArrayList<>(); if (report.getShuffleFileOutputsCount() > 0) { this.getTask().setShuffleFileOutputs(report.getShuffleFileOutputsList()); diff --git a/tajo-core/src/main/java/org/apache/tajo/session/Session.java b/tajo-core/src/main/java/org/apache/tajo/session/Session.java index 0d067dcfef..54aab201f6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/session/Session.java +++ b/tajo-core/src/main/java/org/apache/tajo/session/Session.java @@ -41,7 +41,7 @@ public class Session implements SessionConstants, ProtoObject, Clo private final String userName; private String currentDatabase; private final Map sessionVariables; - private final Map nonForwardQueryMap = new HashMap(); + private final Map nonForwardQueryMap = new HashMap<>(); private LoadingCache cache; // transient status @@ -53,7 +53,7 @@ public Session(String sessionId, String userName, String databaseName) { this.currentDatabase = databaseName; this.lastAccessTime = System.currentTimeMillis(); - this.sessionVariables = new HashMap(); + this.sessionVariables = new HashMap<>(); sessionVariables.put(SessionVars.SESSION_ID.keyname(), sessionId); sessionVariables.put(SessionVars.USERNAME.keyname(), userName); selectDatabase(databaseName); diff --git a/tajo-core/src/main/java/org/apache/tajo/session/SessionManager.java b/tajo-core/src/main/java/org/apache/tajo/session/SessionManager.java index a69533fa1e..38a39ab3cc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/session/SessionManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/session/SessionManager.java @@ -34,7 +34,7 @@ public class SessionManager extends CompositeService implements EventHandler { private static final Log LOG = LogFactory.getLog(SessionManager.class); - public final ConcurrentHashMap sessions = new ConcurrentHashMap(); + public final ConcurrentHashMap sessions = new ConcurrentHashMap<>(); private final Dispatcher dispatcher; private SessionLivelinessMonitor sessionLivelinessMonitor; diff --git a/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java index 919b2c251a..f7928ea6a8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/JSPUtil.java @@ -87,7 +87,7 @@ public static String getTajoMasterHttpAddr(Configuration config) { public static List sortQueryMasterTask(Collection queryMasterTasks, final boolean desc) { - List queryMasterTaskList = new ArrayList(queryMasterTasks); + List queryMasterTaskList = new ArrayList<>(queryMasterTasks); Collections.sort(queryMasterTaskList, new Comparator() { @@ -106,7 +106,7 @@ public int compare(QueryMasterTask task1, QueryMasterTask task2) { public static List sortQueryInProgress(Collection queryInProgresses, final boolean desc) { - List queryProgressList = new ArrayList(queryInProgresses); + List queryProgressList = new ArrayList<>(queryInProgresses); Collections.sort(queryProgressList, new Comparator() { @Override @@ -123,7 +123,7 @@ public int compare(QueryInProgress query1, QueryInProgress query2) { } public static List sortStages(Collection stages) { - List stageList = new ArrayList(stages); + List stageList = new ArrayList<>(stages); Collections.sort(stageList, new Comparator() { @Override public int compare(Stage stage1, Stage stage2) { @@ -146,7 +146,7 @@ public int compare(Stage stage1, Stage stage2) { } public static List sortStageHistories(Collection stages) { - List stageList = new ArrayList(stages); + List stageList = new ArrayList<>(stages); Collections.sort(stageList, new Comparator() { @Override public int compare(StageHistory stage1, StageHistory stage2) { @@ -380,7 +380,7 @@ public static String getPageNavigation(int currentPage, boolean next, String url public static List getPageNavigationList(List originList, int page, int pageSize) { if (originList == null) { - return new ArrayList(); + return new ArrayList<>(); } int start = (page - 1) * pageSize; int end = start + pageSize; diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java index 66077cfdb6..52613354d6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryReader.java @@ -247,13 +247,13 @@ public List getTaskHistory(String queryId, String ebId) throws IOEx public List getTaskHistory(String queryId, String ebId, long startTime) throws IOException { Path queryHistoryFile = getQueryHistoryFilePath(queryId, startTime); if (queryHistoryFile == null) { - return new ArrayList(); + return new ArrayList<>(); } Path detailFile = new Path(queryHistoryFile.getParent(), ebId + HistoryWriter.HISTORY_FILE_POSTFIX); FileSystem fs = HistoryWriter.getNonCrcFileSystem(detailFile, tajoConf); if (!fs.exists(detailFile)) { - return new ArrayList(); + return new ArrayList<>(); } FileStatus fileStatus = fs.getFileStatus(detailFile); diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java index 5fca7a747e..bc62cac115 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/history/HistoryWriter.java @@ -61,9 +61,9 @@ public class HistoryWriter extends AbstractService { public static final String HISTORY_FILE_POSTFIX = ".hist"; private final LinkedBlockingQueue> - historyQueue = new LinkedBlockingQueue>(); + historyQueue = new LinkedBlockingQueue<>(); // key: yyyyMMddHH - private Map taskWriters = new HashMap(); + private Map taskWriters = new HashMap<>(); // For TajoMaster's query list private WriterHolder querySummaryWriter = null; @@ -131,7 +131,7 @@ public void serviceStart() throws Exception { /* asynchronously append to history file */ public WriterFuture appendHistory(History history) { - WriterFuture future = new WriterFuture(history); + WriterFuture future = new WriterFuture<>(history); historyQueue.add(future); return future; } @@ -255,7 +255,7 @@ public void run() { Calendar cal = Calendar.getInstance(); cal.add(Calendar.HOUR_OF_DAY, -2); String closeTargetTime = df.format(cal.getTime()); - List closingTargets = new ArrayList(); + List closingTargets = new ArrayList<>(); for (String eachWriterTime : taskWriters.keySet()) { if (eachWriterTime.compareTo(closeTargetTime) <= 0) { diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java b/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java index fdc45a1dbe..46d333d19d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/history/QueryHistory.java @@ -124,7 +124,7 @@ public QueryHistoryProto getProto() { .setLogicalPlan(logicalPlan) .setDistributedPlan(distributedPlan); - List sessionProtos = new ArrayList(); + List sessionProtos = new ArrayList<>(); if (sessionVariables != null) { KeyValueProto.Builder keyValueBuilder = KeyValueProto.newBuilder(); @@ -139,7 +139,7 @@ public QueryHistoryProto getProto() { builder.addAllSessionVariables(sessionProtos); - List stageHistoryProtos = new ArrayList(); + List stageHistoryProtos = new ArrayList<>(); if (stageHistories != null) { for (StageHistory eachStage: stageHistories) { stageHistoryProtos.add((eachStage.getProto())); diff --git a/tajo-core/src/main/java/org/apache/tajo/util/history/StageHistory.java b/tajo-core/src/main/java/org/apache/tajo/util/history/StageHistory.java index e760f86912..03b7fac0d1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/history/StageHistory.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/history/StageHistory.java @@ -236,7 +236,7 @@ public String toTasksJson() { public static List fromJsonTasks(String json) { if (json == null || json.trim().isEmpty()) { - return new ArrayList(); + return new ArrayList<>(); } return CoreGsonHelper.getInstance().fromJson(json, new TypeToken>() { }.getType()); diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/LogEventGaugeSet.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/LogEventGaugeSet.java index 6e130ffff1..10d17042ca 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/LogEventGaugeSet.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/LogEventGaugeSet.java @@ -29,7 +29,7 @@ public class LogEventGaugeSet implements MetricSet { @Override public Map getMetrics() { - final Map gauges = new HashMap(); + final Map gauges = new HashMap<>(); gauges.put("Fatal", new Gauge() { @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/MetricsFilterList.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/MetricsFilterList.java index b2fc6e4862..e3c2a7895b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/MetricsFilterList.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/MetricsFilterList.java @@ -25,7 +25,7 @@ import java.util.List; public class MetricsFilterList implements MetricFilter { - List filters = new ArrayList(); + List filters = new ArrayList<>(); public void addMetricFilter(MetricFilter filter) { filters.add(filter); diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/RegexpMetricsFilter.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/RegexpMetricsFilter.java index 4faa3e7e45..ef74a4e83e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/RegexpMetricsFilter.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/RegexpMetricsFilter.java @@ -27,7 +27,7 @@ import java.util.regex.Pattern; public class RegexpMetricsFilter implements MetricFilter { - List filterPatterns = new ArrayList(); + List filterPatterns = new ArrayList<>(); public RegexpMetricsFilter(Collection filterExpressions) { for(String eachExpression: filterExpressions) { diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java index ecdf5ef868..8d02ddd9ce 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java @@ -45,7 +45,7 @@ public class TajoSystemMetrics extends TajoMetrics { private String hostAndPort; - private List metricsReporters = new ArrayList(); + private List metricsReporters = new ArrayList<>(); private boolean inited = false; @@ -126,7 +126,7 @@ public void start() { private void setMetricsReporter(String groupName) { // reporter name -> class name - Map reporters = new HashMap(); + Map reporters = new HashMap<>(); List reporterNames = metricsProps.getList(groupName + ".reporters"); if(reporterNames.isEmpty()) { @@ -134,7 +134,7 @@ private void setMetricsReporter(String groupName) { return; } - Map allReporterProperties = new HashMap(); + Map allReporterProperties = new HashMap<>(); Iterator keys = metricsProps.getKeys(); while (keys.hasNext()) { @@ -191,7 +191,7 @@ private void setMetricsReporter(String groupName) { } private Map findMetircsProperties(Map allReporterProperties, String findKey) { - Map metricsProperties = new HashMap(); + Map metricsProperties = new HashMap<>(); for (Map.Entry entry: allReporterProperties.entrySet()) { String eachKey = entry.getKey(); diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/reporter/TajoMetricsReporter.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/reporter/TajoMetricsReporter.java index 584fcbe334..d6e27d3f57 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/reporter/TajoMetricsReporter.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/reporter/TajoMetricsReporter.java @@ -32,10 +32,10 @@ public abstract void report(SortedMap gauges, SortedMap timers); public Map> findMetricsItemGroup(SortedMap metricsMap) { - Map> metricsGroup = new HashMap>(); + Map> metricsGroup = new HashMap<>(); String previousGroup = null; - Map groupItems = new HashMap(); + Map groupItems = new HashMap<>(); for (Map.Entry entry : metricsMap.entrySet()) { String key = entry.getKey(); @@ -66,7 +66,7 @@ public Map> findMetricsItemGroup(SortedMap if (previousGroup != null && !previousGroup.equals(groupName)) { metricsGroup.put(previousGroup, groupItems); - groupItems = new HashMap(); + groupItems = new HashMap<>(); } groupItems.put(itemName, entry.getValue()); previousGroup = groupName; diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/reporter/TajoMetricsScheduledReporter.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/reporter/TajoMetricsScheduledReporter.java index 7e0ec4ab1a..7e3d9bca6e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/reporter/TajoMetricsScheduledReporter.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/reporter/TajoMetricsScheduledReporter.java @@ -103,7 +103,7 @@ public void init(MetricRegistry registry, filterList.addMetricFilter(new GroupNameMetricsFilter(metricsName)); String regexpFilterKey = metricsPropertyKey + "regexp."; - Set regexpExpressions = new HashSet(); + Set regexpExpressions = new HashSet<>(); for(Map.Entry entry: metricsProperties.entrySet()) { String key = entry.getKey(); diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/HttpServer.java b/tajo-core/src/main/java/org/apache/tajo/webapp/HttpServer.java index 60faef26a0..72309eb196 100644 --- a/tajo-core/src/main/java/org/apache/tajo/webapp/HttpServer.java +++ b/tajo-core/src/main/java/org/apache/tajo/webapp/HttpServer.java @@ -51,9 +51,9 @@ public class HttpServer { protected final Connector listener; protected final WebAppContext webAppContext; protected final boolean findPort; - protected final Map defaultContexts = - new HashMap(); - protected final List filterNames = new ArrayList(); + protected final Map defaultContexts = + new HashMap<>(); + protected final List filterNames = new ArrayList<>(); private static final int MAX_RETRIES = 10; private final boolean listenerStartedExternally; static final String STATE_DESCRIPTION_ALIVE = " - alive"; diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java index a04ff6c6b6..896c83e20c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java +++ b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java @@ -71,7 +71,7 @@ public class QueryExecutorServlet extends HttpServlet { //queryRunnerId -> QueryRunner //TODO We must handle the session. - private transient final Map queryRunners = new HashMap(); + private transient final Map queryRunners = new HashMap<>(); private transient TajoConf tajoConf; private transient TajoClient tajoClient; @@ -105,7 +105,7 @@ public void init(ServletConfig config) throws ServletException { public void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { String action = request.getParameter("action"); - Map returnValue = new HashMap(); + Map returnValue = new HashMap<>(); try { if(tajoClient == null) { errorResponse(response, "TajoClient not initialized"); @@ -241,7 +241,7 @@ private void errorResponse(HttpServletResponse response, Exception e) throws IOE } private void errorResponse(HttpServletResponse response, String message) throws IOException { - Map errorMessage = new HashMap(); + Map errorMessage = new HashMap<>(); errorMessage.put("success", "false"); errorMessage.put("errorMessage", message); writeHttpResponse(response, errorMessage); @@ -261,7 +261,7 @@ class QueryRunnerCleaner extends Thread { public void run() { List queryRunnerList; synchronized(queryRunners) { - queryRunnerList = new ArrayList(queryRunners.values()); + queryRunnerList = new ArrayList<>(queryRunners.values()); for(QueryRunner eachQueryRunner: queryRunnerList) { if(!eachQueryRunner.running.get() && (System.currentTimeMillis() - eachQueryRunner.finishTime > 180 * 1000)) { @@ -291,7 +291,7 @@ class QueryRunner extends Thread { AtomicInteger progress = new AtomicInteger(0); - List columnNames = new ArrayList(); + List columnNames = new ArrayList<>(); List> queryResult; @@ -499,7 +499,7 @@ private void MakeResultText(ResultSet res, TableDesc desc) throws SQLException { for(int i = 0; i < numOfColumns; i++) { columnNames.add(rsmd.getColumnName(i + 1)); } - queryResult = new ArrayList>(); + queryResult = new ArrayList<>(); if(sizeLimit < resultRows) { numOfRows = (long)((float)(resultRows) * ((float)sizeLimit / (float) resultRows)); @@ -512,7 +512,7 @@ private void MakeResultText(ResultSet res, TableDesc desc) throws SQLException { if(rowCount > numOfRows) { break; } - List row = new ArrayList(); + List row = new ArrayList<>(); for(int i = 0; i < numOfColumns; i++) { row.add(String.valueOf(res.getObject(i + 1))); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java index a3cc8fc02c..db28433cdf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java @@ -251,7 +251,7 @@ public void fatalError(TaskAttemptId taskAttemptId, String message) { try { //If QueryMaster does not responding, current execution block should be stop - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); getStub().fatalError(callFuture.getController(), builder.build(), callFuture); callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); } catch (Exception e) { @@ -298,7 +298,7 @@ private void sendHashShuffleReport(ExecutionBlockId ebId) throws Exception { if (shuffles == null) { reporterBuilder.addAllIntermediateEntries(intermediateEntries); - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); stub.doneExecutionBlock(callFuture.getController(), reporterBuilder.build(), callFuture); callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); return; @@ -353,7 +353,7 @@ private void sendHashShuffleReport(ExecutionBlockId ebId) throws Exception { } } try { - CallFuture callFuture = new CallFuture(); + CallFuture callFuture = new CallFuture<>(); stub.doneExecutionBlock(callFuture.getController(), reporterBuilder.build(), callFuture); callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); } catch (Throwable e) { @@ -391,7 +391,7 @@ public void run() { if(tasks.size() == 0){ masterStub.ping(null, getExecutionBlockId().getProto(), NullCallback.get()); } else { - for (Task task : new ArrayList(tasks.values())){ + for (Task task : new ArrayList<>(tasks.values())){ if (task.getTaskContext().getState() == TajoProtos.TaskAttemptState.TA_RUNNING && task.isProgressChanged()) { diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockSharedResource.java b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockSharedResource.java index 1a83104fae..d55c4bacf0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockSharedResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockSharedResource.java @@ -96,7 +96,7 @@ public EvalNode compileEval(Schema schema, EvalNode eval) { public EvalNode getPreCompiledEval(Schema schema, EvalNode eval) { if (codeGenEnabled) { - Pair key = new Pair(schema, eval); + Pair key = new Pair<>(schema, eval); if (compilationContext.getPrecompiedEvals().containsKey(key)) { return compilationContext.getPrecompiedEvals().get(key); } else { diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java index 07a9ba6ac6..3fa9d11161 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java @@ -53,8 +53,8 @@ public class FetchImpl implements ProtoObject, Cloneable { private long length = -1; public FetchImpl() { - taskIds = new ArrayList(); - attemptIds = new ArrayList(); + taskIds = new ArrayList<>(); + attemptIds = new ArrayList<>(); } public FetchImpl(FetchProto proto) { diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java index a3b71e1cc0..3ca0bfb486 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/NodeStatusUpdater.java @@ -165,7 +165,7 @@ protected NodeHeartbeatResponse sendHeartbeat(NodeHeartbeatRequest requestProto) NodeHeartbeatResponse response = null; try { - CallFuture callBack = new CallFuture(); + CallFuture callBack = new CallFuture<>(); resourceTracker.nodeHeartbeat(callBack.getController(), requestProto, callBack); response = callBack.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java index 607e7ff05b..0071a145d8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java @@ -493,7 +493,7 @@ public void cleanupTemporalDirectories() { for (Path path : iter) { PathData[] items = PathData.expandAsGlob(localFS.makeQualified(new Path(path, "*")).toString(), systemConf); - ArrayList paths = new ArrayList(); + ArrayList paths = new ArrayList<>(); for (PathData pd : items) { paths.add(pd.path); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index 228c32ad84..485fa03379 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -106,7 +106,7 @@ public TaskAttemptContext(QueryContext queryContext, final ExecutionBlockContext if (fragmentMap.containsKey(t.getId())) { fragmentMap.get(t.getId()).add(t); } else { - List frags = new ArrayList(); + List frags = new ArrayList<>(); frags.add(t); fragmentMap.put(t.getId(), frags); } @@ -259,7 +259,7 @@ public void updateAssignedFragments(String tableId, Fragment[] fragments) { if (fragmentMap.containsKey(t.getTableName())) { fragmentMap.get(t.getTableName()).add(t.getProto()); } else { - List frags = new ArrayList(); + List frags = new ArrayList<>(); frags.add(t.getProto()); fragmentMap.put(t.getTableName(), frags); } @@ -273,7 +273,7 @@ public void addFragments(String tableId, FragmentProto[] fragments) { List tableFragments = fragmentMap.get(tableId); if (tableFragments == null) { - tableFragments = new ArrayList(); + tableFragments = new ArrayList<>(); } List paths = fragmentToPath(tableFragments); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java index 1d6e2b8356..8b9d43fd51 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java @@ -66,7 +66,7 @@ public TaskExecutor(TajoWorker.WorkerContext workerContext) { this.workerContext = workerContext; this.allocatedResourceMap = Maps.newConcurrentMap(); this.runningTasks = new AtomicInteger(); - this.taskQueue = new LinkedBlockingQueue(); + this.taskQueue = new LinkedBlockingQueue<>(); this.fetcherThreadPoolList = Lists.newArrayList(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java index a8e874c029..a96e3bb6c1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskImpl.java @@ -365,7 +365,7 @@ private void waitForFetch() throws InterruptedException, IOException { Collection inputs = Lists.newArrayList(context.getInputTables()); // Get all broadcasted tables - Set broadcastTableNames = new HashSet(); + Set broadcastTableNames = new HashSet<>(); List broadcasts = context.getEnforcer().getEnforceProperties(EnforceType.BROADCAST); if (broadcasts != null) { for (EnforceProperty eachBroadcast : broadcasts) { @@ -553,7 +553,7 @@ private FileFragment[] localizeFetchedData(File file, String name, TableMeta met FileSystem fs = FileSystem.get(c); Path tablePath = new Path(file.getAbsolutePath()); - List listTablets = new ArrayList(); + List listTablets = new ArrayList<>(); FileFragment tablet; FileStatus[] fileLists = fs.listStatus(tablePath); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java index a0b3f97f17..e5cf7c92fb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskManager.java @@ -122,7 +122,7 @@ protected ExecutionBlockContext createExecutionBlock(ExecutionBlockId executionB client = RpcClientManager.getInstance().newClient(address, QueryMasterProtocol.class, true, rpcParams); QueryMasterProtocol.QueryMasterProtocolService.Interface stub = client.getStub(); - CallFuture callback = new CallFuture(); + CallFuture callback = new CallFuture<>(); stub.getExecutionBlockContext(callback.getController(), request.build(), callback); ExecutionBlockContextResponse contextProto = diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/rule/WorkerRuleProvider.java b/tajo-core/src/main/java/org/apache/tajo/worker/rule/WorkerRuleProvider.java index 81c2522bcc..1b58c32591 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/rule/WorkerRuleProvider.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/rule/WorkerRuleProvider.java @@ -37,7 +37,7 @@ public class WorkerRuleProvider implements SelfDiagnosisRuleProvider { public List getDefinedRules() { Set classSet = ClassUtil.findClasses(SelfDiagnosisRule.class, getClass().getPackage().getName()); - List ruleList = new ArrayList(classSet.size()); + List ruleList = new ArrayList<>(classSet.size()); for (Class ruleClazz: classSet) { try { diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/ClientApplication.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/ClientApplication.java index b88b8ed0e5..7d7c9240fa 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/ClientApplication.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/ClientApplication.java @@ -56,7 +56,7 @@ public ClientApplication(MasterContext masterContext) { this.secureRandom = new SecureRandom(); - this.queryIdToResultSetCacheIdMap = new ConcurrentHashMap(); + this.queryIdToResultSetCacheIdMap = new ConcurrentHashMap<>(); this.queryResultScannerCache = CacheBuilder.newBuilder() .concurrencyLevel(4) .maximumSize(1000) @@ -66,7 +66,7 @@ public ClientApplication(MasterContext masterContext) { @Override public Set> getClasses() { - Set> classes = new HashSet>(); + Set> classes = new HashSet<>(); classes.add(SessionsResource.class); classes.add(DatabasesResource.class); diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/JerseyResourceDelegateContext.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/JerseyResourceDelegateContext.java index 9f1fb4ac48..69c455047d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/JerseyResourceDelegateContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/JerseyResourceDelegateContext.java @@ -28,7 +28,7 @@ public class JerseyResourceDelegateContext { private final ConcurrentMap, Object> contextMap = - new ConcurrentHashMap, Object>(); + new ConcurrentHashMap<>(); /** * Add value to Context. If value exists, it will overwrite. diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/JerseyResourceDelegateContextKey.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/JerseyResourceDelegateContextKey.java index 3c059d45ea..b1d759a82e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/JerseyResourceDelegateContextKey.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/JerseyResourceDelegateContextKey.java @@ -26,7 +26,7 @@ public class JerseyResourceDelegateContextKey { private static final ConcurrentMap> keyMap = - new ConcurrentHashMap>(); + new ConcurrentHashMap<>(); private final String name; private final Class type; @@ -43,7 +43,7 @@ public static JerseyResourceDelegateContextKey valueOf(String name, Class JerseyResourceDelegateContextKey key = (JerseyResourceDelegateContextKey) keyMap.get(name); if (key == null) { - key = new JerseyResourceDelegateContextKey(name, type); + key = new JerseyResourceDelegateContextKey<>(name, type); keyMap.putIfAbsent(name, key); } diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/ClusterResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/ClusterResource.java index 19d8b28685..9311139c41 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/ClusterResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/ClusterResource.java @@ -100,13 +100,13 @@ public Response run(JerseyResourceDelegateContext context) { MasterContext masterContext = context.get(masterContextKey); Map workerMap = masterContext.getResourceManager().getNodes(); - List workerList = new ArrayList(); + List workerList = new ArrayList<>(); for (NodeStatus nodeStatus : workerMap.values()) { workerList.add(new WorkerResponse(nodeStatus)); } - Map> workerResponseMap = new HashMap>(); + Map> workerResponseMap = new HashMap<>(); workerResponseMap.put(workersName, workerList); return Response.ok(workerResponseMap).build(); diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/DatabasesResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/DatabasesResource.java index 046e538cb5..5d3c95b430 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/DatabasesResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/DatabasesResource.java @@ -105,7 +105,7 @@ public Response run(JerseyResourceDelegateContext context) { MasterContext masterContext = context.get(masterContextKey); Collection databaseNames = masterContext.getCatalog().getAllDatabaseNames(); - Map> databaseNamesMap = new HashMap>(); + Map> databaseNamesMap = new HashMap<>(); databaseNamesMap.put(databasesKeyName, databaseNames); return Response.ok(databaseNamesMap).build(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java index ead4b71e30..871328d66d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/FunctionsResource.java @@ -89,8 +89,8 @@ public Response run(JerseyResourceDelegateContext context) { Collection functionDescriptors = masterContext.getCatalog().getFunctions(); if (functionDescriptors.size() > 0) { - List functionSignature = - new ArrayList(functionDescriptors.size()); + List functionSignature = + new ArrayList<>(functionDescriptors.size()); for (FunctionDesc functionDesc : functionDescriptors) { functionSignature.add(functionDesc.getSignature()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java index eb67167b1f..e4cf025a5e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/QueryResource.java @@ -145,8 +145,8 @@ public Response run(JerseyResourceDelegateContext context) { return ResourcesUtil.createBadRequestResponse(LOG, state + " is not a valid query state."); } - Map> queriesMap = new HashMap>(); - List queriesInfo = new ArrayList(); + Map> queriesMap = new HashMap<>(); + List queriesInfo = new ArrayList<>(); QueryManager queryManager = masterContext.getQueryJobManager(); for (QueryInProgress queryInProgress: queryManager.getSubmittedQueries()) { @@ -172,7 +172,7 @@ public Response run(JerseyResourceDelegateContext context) { } private List selectQueriesInfoByState(List queriesInfo, TajoProtos.QueryState state) { - List resultQueriesInfo = new ArrayList(queriesInfo.size()/2); + List resultQueriesInfo = new ArrayList<>(queriesInfo.size() / 2); for (QueryInfo queryInfo: queriesInfo) { if (state.equals(queryInfo.getQueryState())) { @@ -184,7 +184,7 @@ private List selectQueriesInfoByState(List queriesInfo, Ta } private List selectQueriesInfoByTime(List queriesInfo, long startTime, long endTime) { - List resultQueriesInfo = new ArrayList(queriesInfo.size()/2); + List resultQueriesInfo = new ArrayList<>(queriesInfo.size() / 2); for (QueryInfo queryInfo: queriesInfo) { if (queryInfo.getStartTime() > startTime) { diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/SessionsResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/SessionsResource.java index 8456202d35..52d8fe77e7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/SessionsResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/SessionsResource.java @@ -267,11 +267,11 @@ public Response run(JerseyResourceDelegateContext context) { String sessionId = context.get(sessionIdKey); try { - Map> variablesMap = new HashMap>(); + Map> variablesMap = new HashMap<>(); variablesMap.put(variablesOutputKeyName, masterContext.getSessionManager().getAllVariables(sessionId)); GenericEntity>> variablesEntity = - new GenericEntity>>(variablesMap, Map.class); + new GenericEntity<>(variablesMap, Map.class); return Response.ok(variablesEntity).build(); } catch (InvalidSessionException e) { LOG.error("Unable to find a session : " + sessionId); diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/TablesResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/TablesResource.java index 3dba02f8ad..3afe7c812b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/TablesResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/TablesResource.java @@ -121,7 +121,7 @@ public Response run(JerseyResourceDelegateContext context) { } Collection tableNames = catalogService.getAllTableNames(databaseName); - Map> tableNamesMap = new HashMap>(); + Map> tableNamesMap = new HashMap<>(); tableNamesMap.put(tablesKeyName, tableNames); return Response.ok(tableNamesMap).build(); } diff --git a/tajo-core/src/main/resources/webapps/admin/cluster.jsp b/tajo-core/src/main/resources/webapps/admin/cluster.jsp index 7aad8d4e29..fd942bdda3 100644 --- a/tajo-core/src/main/resources/webapps/admin/cluster.jsp +++ b/tajo-core/src/main/resources/webapps/admin/cluster.jsp @@ -39,17 +39,17 @@ String masterLabel = socketAddress.getAddress().getHostName()+ ":" + socketAddress.getPort(); Map nodes = master.getContext().getResourceManager().getNodes(); - List wokerKeys = new ArrayList(nodes.keySet()); + List wokerKeys = new ArrayList<>(nodes.keySet()); Collections.sort(wokerKeys); int runningQueryMasterTasks = 0; - Set liveNodes = new TreeSet(); - Set deadNodes = new TreeSet(); - Set decommissionNodes = new TreeSet(); + Set liveNodes = new TreeSet<>(); + Set deadNodes = new TreeSet<>(); + Set decommissionNodes = new TreeSet<>(); - Set liveQueryMasters = new TreeSet(); - Set deadQueryMasters = new TreeSet(); + Set liveQueryMasters = new TreeSet<>(); + Set deadQueryMasters = new TreeSet<>(); for(NodeStatus eachNode: nodes.values()) { liveQueryMasters.add(eachNode); diff --git a/tajo-core/src/main/resources/webapps/admin/functions.jsp b/tajo-core/src/main/resources/webapps/admin/functions.jsp index cf3ddc5a9f..2006ef52bc 100644 --- a/tajo-core/src/main/resources/webapps/admin/functions.jsp +++ b/tajo-core/src/main/resources/webapps/admin/functions.jsp @@ -28,7 +28,7 @@ TajoMaster master = (TajoMaster) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object"); CatalogService catalog = master.getCatalog(); - List functions = new ArrayList(catalog.getFunctions()); + List functions = new ArrayList<>(catalog.getFunctions()); Collections.sort(functions); %> diff --git a/tajo-core/src/main/resources/webapps/admin/query.jsp b/tajo-core/src/main/resources/webapps/admin/query.jsp index bcf7766bb8..e44ac8cf5f 100644 --- a/tajo-core/src/main/resources/webapps/admin/query.jsp +++ b/tajo-core/src/main/resources/webapps/admin/query.jsp @@ -39,11 +39,11 @@ String masterLabel = socketAddress.getAddress().getHostName()+ ":" + socketAddress.getPort(); List submittedQueries = - new ArrayList(master.getContext().getQueryJobManager().getSubmittedQueries()); + new ArrayList<>(master.getContext().getQueryJobManager().getSubmittedQueries()); JSPUtil.sortQueryInProgress(submittedQueries, true); List runningQueries = - new ArrayList(master.getContext().getQueryJobManager().getRunningQueries()); + new ArrayList<>(master.getContext().getQueryJobManager().getRunningQueries()); JSPUtil.sortQueryInProgress(runningQueries, true); int currentPage = 1; @@ -64,7 +64,7 @@ SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); Map workers = master.getContext().getResourceManager().getNodes(); - Map portMap = new HashMap(); + Map portMap = new HashMap<>(); Collection queryMasters = master.getContext().getResourceManager().getQueryMasters(); if (queryMasters == null || queryMasters.isEmpty()) { diff --git a/tajo-core/src/main/resources/webapps/admin/querytasks.jsp b/tajo-core/src/main/resources/webapps/admin/querytasks.jsp index a7b26ae1d8..f2dc68b190 100644 --- a/tajo-core/src/main/resources/webapps/admin/querytasks.jsp +++ b/tajo-core/src/main/resources/webapps/admin/querytasks.jsp @@ -76,7 +76,7 @@ Collection allWorkers = master.getContext().getResourceManager().getNodes().values(); - Map nodeMap = new HashMap(); + Map nodeMap = new HashMap<>(); if(allWorkers != null) { for(NodeStatus eachWorker: allWorkers) { nodeMap.put(eachWorker.getConnectionInfo().getHostAndPeerRpcPort(), eachWorker); @@ -189,7 +189,7 @@ <% - List filteredTasks = new ArrayList(); + List filteredTasks = new ArrayList<>(); for(TaskHistory eachTask: allTasks) { if (!"ALL".equals(status)) { if (!status.equals(eachTask.getState().toString())) { diff --git a/tajo-core/src/main/resources/webapps/worker/queryplan.jsp b/tajo-core/src/main/resources/webapps/worker/queryplan.jsp index 422edfc716..891822421c 100644 --- a/tajo-core/src/main/resources/webapps/worker/queryplan.jsp +++ b/tajo-core/src/main/resources/webapps/worker/queryplan.jsp @@ -46,7 +46,7 @@ Query query = queryMasterTask.getQuery(); - Map stageMap = new HashMap(); + Map stageMap = new HashMap<>(); for(Stage eachStage : query.getStages()) { stageMap.put(eachStage.getId(), eachStage); @@ -102,7 +102,7 @@ String curIdStr = null; int x=35, y=1; int pos; - List stageInfos = new ArrayList(); + List stageInfos = new ArrayList<>(); stageInfos.add(new StageInfo(masterPlan.getRoot(), null, null, x, y, 0)); diff --git a/tajo-core/src/main/resources/webapps/worker/querytasks.jsp b/tajo-core/src/main/resources/webapps/worker/querytasks.jsp index caaec35cc0..d5e9a9f519 100644 --- a/tajo-core/src/main/resources/webapps/worker/querytasks.jsp +++ b/tajo-core/src/main/resources/webapps/worker/querytasks.jsp @@ -64,7 +64,7 @@ List allWorkers = tajoWorker.getWorkerContext() .getQueryMasterManagerService().getQueryMaster().getAllWorker(); - Map workerMap = new HashMap(); + Map workerMap = new HashMap<>(); if(allWorkers != null) { for(TajoProtos.WorkerConnectionInfoProto eachWorker: allWorkers) { workerMap.put(eachWorker.getId(), eachWorker); @@ -204,7 +204,7 @@ <% - List filteredTask = new ArrayList(); + List filteredTask = new ArrayList<>(); for(Task eachTask : allTasks) { if (!"ALL".equals(status)) { if (!status.equals(eachTask.getLastAttemptStatus().toString())) { diff --git a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java index f22bafeed2..ad0fd3c1f3 100644 --- a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java +++ b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java @@ -364,7 +364,7 @@ private String convertPattern(final String pattern) { @Override public ResultSet getTables(@Nullable String catalog, @Nullable String schemaPattern, @Nullable String tableNamePattern, @Nullable String [] types) throws SQLException { - final List resultTables = new ArrayList(); + final List resultTables = new ArrayList<>(); String regtableNamePattern = convertPattern(tableNamePattern == null ? null : tableNamePattern); List targetCatalogs = Lists.newArrayList(); @@ -440,7 +440,7 @@ public ResultSet getCatalogs() throws SQLException { databaseNames = conn.getCatalogAdminClient().getAllDatabaseNames(); - List tuples = new ArrayList(); + List tuples = new ArrayList<>(); for (String databaseName : databaseNames) { MetaDataTuple tuple = new MetaDataTuple(1); tuple.put(0, new TextDatum(databaseName)); @@ -455,7 +455,7 @@ public ResultSet getCatalogs() throws SQLException { @Override public ResultSet getTableTypes() throws SQLException { - List columns = new ArrayList(); + List columns = new ArrayList<>(); MetaDataTuple tuple = new MetaDataTuple(2); tuple.put(0, new TextDatum("TABLE")); columns.add(tuple); @@ -471,7 +471,7 @@ public ResultSet getTableTypes() throws SQLException { @Override public ResultSet getUDTs(String catalog, String schemaPattern, String typeNamePattern, int[] types) throws SQLException { - List columns = new ArrayList(); + List columns = new ArrayList<>(); return new TajoMetaDataResultSet( Arrays.asList("TYPE_CAT", "TYPE_SCHEM", "TYPE_NAME", "CLASS_NAME", "DATA_TYPE" @@ -490,7 +490,7 @@ public ResultSet getColumns(@Nullable String catalog, @Nullable String schemaPat targetCatalogs.add(catalog); } - List columns = new ArrayList(); + List columns = new ArrayList<>(); try { if (targetCatalogs.isEmpty()) { targetCatalogs.addAll(conn.getCatalogAdminClient().getAllDatabaseNames()); @@ -641,7 +641,7 @@ public ResultSet getCrossReference(String parentCatalog, String parentSchema, St @Override public ResultSet getTypeInfo() throws SQLException { - List tuples = new ArrayList(); + List tuples = new ArrayList<>(); for (Datum[] eachDatums: TajoTypeUtil.getTypeInfos()) { MetaDataTuple tuple = new MetaDataTuple(eachDatums.length); for (int i = 0; i < eachDatums.length; i++) { diff --git a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoPreparedStatement.java b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoPreparedStatement.java index 6b47f9717c..3793bb0669 100644 --- a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoPreparedStatement.java +++ b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoPreparedStatement.java @@ -37,7 +37,7 @@ public class TajoPreparedStatement extends TajoStatement implements PreparedStat /** * save the SQL parameters {paramLoc:paramValue} */ - private final HashMap parameters=new HashMap(); + private final HashMap parameters= new HashMap<>(); /** * keep the current ResultRet update count diff --git a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/util/QueryStringDecoder.java b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/util/QueryStringDecoder.java index 9ec934040d..86d1b1a160 100644 --- a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/util/QueryStringDecoder.java +++ b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/util/QueryStringDecoder.java @@ -59,7 +59,7 @@ private void splitUri() { } protected void decodeParams() throws MalformedURLException, UnsupportedEncodingException { - params = new HashMap>(); + params = new HashMap<>(); String queries = getQueries(); if (queries != null && !queries.isEmpty()) { @@ -112,7 +112,7 @@ protected void addParameter(String name, String value) { List valueList = params.get(name); if (valueList == null) { - valueList = new ArrayList(); + valueList = new ArrayList<>(); params.put(name, valueList); } diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoDatabaseMetaData.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoDatabaseMetaData.java index 8ee6755e11..3107af96a2 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoDatabaseMetaData.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoDatabaseMetaData.java @@ -43,7 +43,7 @@ public static void setUp() throws Exception { } public static List getListFromResultSet(ResultSet resultSet, String columnName) throws SQLException { - List list = new ArrayList(); + List list = new ArrayList<>(); while(resultSet.next()) { list.add(resultSet.getString(columnName)); } @@ -202,7 +202,7 @@ public void testGetTablesWithPattern() throws Exception { TajoConstants.DEFAULT_DATABASE_NAME); Connection conn = DriverManager.getConnection(connUri); - Map> tables = new HashMap>(); + Map> tables = new HashMap<>(); assertDatabaseNotExists("db_1"); executeString("CREATE DATABASE db_1"); assertDatabaseExists("db_1"); @@ -277,7 +277,7 @@ public void testGetColumnsWithPattern() throws Exception { // db.tb, i = {1,2}, 0 <= j < 2 // db.table_, i = {1,2}, 0 <= j < 2 - Map> tables = new HashMap>(); + Map> tables = new HashMap<>(); for (int j = 1; j <= 2; j++) { String dbName = "db" + j; assertDatabaseNotExists(dbName); diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index f8d5c4576b..d29291e5a2 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -307,9 +307,9 @@ public void testDatabaseMetaDataGetTable() throws Exception { int numCols = rsmd.getColumnCount(); assertEquals(5, numCols); - Set retrivedViaJavaAPI = new HashSet(client.getTableList("default")); + Set retrivedViaJavaAPI = new HashSet<>(client.getTableList("default")); - Set retrievedViaJDBC = new HashSet(); + Set retrievedViaJDBC = new HashSet<>(); while (rs.next()) { retrievedViaJDBC.add(rs.getString("TABLE_NAME")); } diff --git a/tajo-maven-plugins/src/main/java/org/apache/tajo/maven/plugin/protoc/ProtocMojo.java b/tajo-maven-plugins/src/main/java/org/apache/tajo/maven/plugin/protoc/ProtocMojo.java index 498f85d2bd..e42ac6b537 100644 --- a/tajo-maven-plugins/src/main/java/org/apache/tajo/maven/plugin/protoc/ProtocMojo.java +++ b/tajo-maven-plugins/src/main/java/org/apache/tajo/maven/plugin/protoc/ProtocMojo.java @@ -58,11 +58,11 @@ public void execute() throws MojoExecutionException { if (protocCommand == null || protocCommand.trim().isEmpty()) { protocCommand = "protoc"; } - List command = new ArrayList(); + List command = new ArrayList<>(); command.add(protocCommand); command.add("--version"); Exec exec = new Exec(this); - List out = new ArrayList(); + List out = new ArrayList<>(); if (exec.run(command, out) == 127) { getLog().error("protoc, not found at: " + protocCommand); throw new MojoExecutionException("protoc failure"); @@ -85,7 +85,7 @@ public void execute() throws MojoExecutionException { output); } } - command = new ArrayList(); + command = new ArrayList<>(); command.add(protocCommand); command.add("--java_out=" + output.getCanonicalPath()); if (imports != null) { @@ -97,7 +97,7 @@ public void execute() throws MojoExecutionException { command.add(f.getCanonicalPath()); } exec = new Exec(this); - out = new ArrayList(); + out = new ArrayList<>(); if (exec.run(command, out) != 0) { getLog().error("protoc compiler error"); for (String s : out) { diff --git a/tajo-maven-plugins/src/main/java/org/apache/tajo/maven/plugin/util/Exec.java b/tajo-maven-plugins/src/main/java/org/apache/tajo/maven/plugin/util/Exec.java index 89ba6fb491..c657cbfed8 100644 --- a/tajo-maven-plugins/src/main/java/org/apache/tajo/maven/plugin/util/Exec.java +++ b/tajo-maven-plugins/src/main/java/org/apache/tajo/maven/plugin/util/Exec.java @@ -88,7 +88,7 @@ private static class OutputBufferThread extends Thread { */ public OutputBufferThread(InputStream is) { this.setDaemon(true); - output = new ArrayList(); + output = new ArrayList<>(); reader = new BufferedReader(new InputStreamReader(is)); } diff --git a/tajo-maven-plugins/src/main/java/org/apache/tajo/maven/plugin/versioninfo/VersionInfoMojo.java b/tajo-maven-plugins/src/main/java/org/apache/tajo/maven/plugin/versioninfo/VersionInfoMojo.java index 5347f4055a..44db8ee277 100644 --- a/tajo-maven-plugins/src/main/java/org/apache/tajo/maven/plugin/versioninfo/VersionInfoMojo.java +++ b/tajo-maven-plugins/src/main/java/org/apache/tajo/maven/plugin/versioninfo/VersionInfoMojo.java @@ -115,7 +115,7 @@ private String getBuildTime() { private SCM determineSCM() throws Exception { Exec exec = new Exec(this); SCM scm = SCM.NONE; - scmOut = new ArrayList(); + scmOut = new ArrayList<>(); int ret = exec.run(Arrays.asList(svnCommand, "info"), scmOut); if (ret == 0) { scm = SCM.SVN; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java index 89089e0c5c..f4c11cd013 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java @@ -119,7 +119,7 @@ private static Pair convertTypesIfNecessary(Context ctx, Eva // If one of both is NULL, it just returns the original types without casting. if (lhsType == Type.NULL_TYPE || rhsType == Type.NULL_TYPE) { - return new Pair(lhs, rhs); + return new Pair<>(lhs, rhs); } Type toBeCasted = TUtil.getFromNestedMap(CatalogUtil.OPERATION_CASTING_MAP, lhsType, rhsType); @@ -133,7 +133,7 @@ private static Pair convertTypesIfNecessary(Context ctx, Eva } } - return new Pair(lhs, rhs); + return new Pair<>(lhs, rhs); } /** diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprNormalizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprNormalizer.java index ff7eabda9d..ab1914ba3a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprNormalizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprNormalizer.java @@ -91,9 +91,9 @@ public static class ExprNormalizedResult { Expr baseExpr; // outmost expressions, which can includes one or more references of the results of aggregation // function. - List aggExprs = new ArrayList(); // aggregation functions - List scalarExprs = new ArrayList(); // scalar expressions which can be referred - List windowAggExprs = new ArrayList(); // window expressions which can be referred + List aggExprs = new ArrayList<>(); // aggregation functions + List scalarExprs = new ArrayList<>(); // scalar expressions which can be referred + List windowAggExprs = new ArrayList<>(); // window expressions which can be referred Set windowSpecs = Sets.newLinkedHashSet(); public ExprNormalizedResult(LogicalPlanner.PlanContext context, boolean tryBinaryCommonTermsElimination) { @@ -118,7 +118,7 @@ public ExprNormalizedResult normalize(LogicalPlanner.PlanContext context, Expr e public ExprNormalizedResult normalize(LogicalPlanner.PlanContext context, Expr expr, boolean subexprElimination) throws TajoException { ExprNormalizedResult exprNormalizedResult = new ExprNormalizedResult(context, subexprElimination); - Stack stack = new Stack(); + Stack stack = new Stack<>(); stack.push(expr); visit(exprNormalizedResult, new Stack(), expr); exprNormalizedResult.baseExpr = stack.pop(); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index 96617d1946..ecb478cb5a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -87,7 +87,7 @@ public LogicalNode optimize(OverridableConf context, LogicalPlan plan) throws Ta rulesBeforeJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog)); DirectedGraphCursor blockCursor = - new DirectedGraphCursor(plan.getQueryBlockGraph(), plan.getRootBlock().getName()); + new DirectedGraphCursor<>(plan.getQueryBlockGraph(), plan.getRootBlock().getName()); if (context == null || context.getBool(SessionVars.TEST_JOIN_OPT_ENABLED)) { // default is true @@ -121,7 +121,7 @@ private void optimizeJoinOrder(LogicalPlan plan, String blockName) throws TajoEx JoinNode old = PlannerUtil.findTopNode(block.getRoot(), NodeType.JOIN); JoinTargetCollector collector = new JoinTargetCollector(); - Set targets = new LinkedHashSet(); + Set targets = new LinkedHashSet<>(); collector.visitJoin(targets, plan, block, old, new Stack()); if (targets.size() == 0) { @@ -158,9 +158,9 @@ private static LogicalNode handleRemainingFiltersIfNecessary(JoinGraphContext jo JoinNode newJoinNode) { // Gather filters from remaining join edges Collection joinEdges = joinGraphContext.getJoinGraph().getEdgesAll(); - Collection markAsEvaluated = new HashSet(joinGraphContext.getEvaluatedJoinConditions()); + Collection markAsEvaluated = new HashSet<>(joinGraphContext.getEvaluatedJoinConditions()); markAsEvaluated.addAll(joinGraphContext.getEvaluatedJoinFilters()); - Set remainingQuals = new HashSet(joinGraphContext.getCandidateJoinFilters()); + Set remainingQuals = new HashSet<>(joinGraphContext.getCandidateJoinFilters()); for (JoinEdge eachEdge : joinEdges) { for (EvalNode eachQual : eachEdge.getJoinQual()) { if (!markAsEvaluated.contains(eachQual)) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java index b7df81054a..457cb2c69e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java @@ -64,11 +64,11 @@ public class LogicalPlan { private Integer noNameColumnId = 0; /** a map from between a block name to a block plan */ - private Map queryBlocks = new LinkedHashMap(); - private Map nodeMap = new HashMap(); - private Map queryBlockByPID = new HashMap(); + private Map queryBlocks = new LinkedHashMap<>(); + private Map nodeMap = new HashMap<>(); + private Map queryBlockByPID = new HashMap<>(); private Map exprToBlockNameMap = TUtil.newHashMap(); - private SimpleDirectedGraph queryBlockGraph = new SimpleDirectedGraph(); + private SimpleDirectedGraph queryBlockGraph = new SimpleDirectedGraph<>(); /** planning and optimization log */ private List planingHistory = Lists.newArrayList(); @@ -266,7 +266,7 @@ public QueryBlock getBlock(LogicalNode node) { public void removeBlock(QueryBlock block) { queryBlocks.remove(block.getName()); - List tobeRemoved = new ArrayList(); + List tobeRemoved = new ArrayList<>(); for (Map.Entry entry : queryBlockByPID.entrySet()) { tobeRemoved.add(entry.getKey()); } @@ -335,7 +335,7 @@ public String getQueryGraphAsString() { } } DirectedGraphCursor cursor = - new DirectedGraphCursor(queryBlockGraph, getRootBlock().getName()); + new DirectedGraphCursor<>(queryBlockGraph, getRootBlock().getName()); while(cursor.hasNext()) { QueryBlock block = getBlock(cursor.nextBlock()); if (block.getPlanHistory().size() > 0) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 3531ce7a84..7593a9b789 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -397,8 +397,8 @@ public boolean isMatch(Expr expr) { }); addNamedExprs(block, referenceNames, normalizedExprList, windowSpecReferencesList, projection, targetsIds); - return new Pair(referenceNames, - windowSpecReferencesList.toArray(new ExprNormalizer.WindowSpecReferences[windowSpecReferencesList.size()])); + return new Pair<>(referenceNames, + windowSpecReferencesList.toArray(new ExprNormalizer.WindowSpecReferences[windowSpecReferencesList.size()])); } private interface Matcher { @@ -407,7 +407,7 @@ private interface Matcher { public List normalize(PlanContext context, Projection projection, ExprNormalizedResult [] normalizedExprList, Matcher matcher) throws TajoException { - List targetIds = new ArrayList(); + List targetIds = new ArrayList<>(); for (int i = 0; i < projection.size(); i++) { NamedExpr namedExpr = projection.getNamedExprs()[i]; @@ -653,8 +653,8 @@ private LogicalNode insertWindowAggNode(PlanContext context, LogicalNode child, windowAggNode.setInSchema(child.getOutSchema()); } - List winFuncRefs = new ArrayList(); - List winFuncs = new ArrayList(); + List winFuncRefs = new ArrayList<>(); + List winFuncs = new ArrayList<>(); List rawWindowSpecs = Lists.newArrayList(); for (Iterator it = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); it.hasNext();) { NamedExpr rawTarget = it.next(); @@ -788,8 +788,8 @@ private LogicalNode insertGroupbyNode(PlanContext context, LogicalNode child, St groupbyNode.setGroupingColumns(new Column[] {}); - Set aggEvalNames = new LinkedHashSet(); - Set aggEvals = new LinkedHashSet(); + Set aggEvalNames = new LinkedHashSet<>(); + Set aggEvals = new LinkedHashSet<>(); boolean includeDistinctFunction = false; for (Iterator it = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); it.hasNext();) { NamedExpr rawTarget = it.next(); @@ -1344,7 +1344,7 @@ public ScanNode visitRelation(PlanContext context, Stack stack, Relation e // Find expression which can be evaluated at this relation node. // Except for column references, additional expressions used in select list, where clause, order-by clauses // can be evaluated here. Their reference names are kept in newlyEvaluatedExprsRef. - Set newlyEvaluatedExprsReferences = new LinkedHashSet(); + Set newlyEvaluatedExprsReferences = new LinkedHashSet<>(); for (Iterator iterator = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); iterator.hasNext();) { NamedExpr rawTarget = iterator.next(); try { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/PlanString.java b/tajo-plan/src/main/java/org/apache/tajo/plan/PlanString.java index b06c68f7f1..d80cb7e0b4 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/PlanString.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/PlanString.java @@ -27,8 +27,8 @@ public class PlanString { final StringBuilder title; - final List explanations = new ArrayList(); - final List details = new ArrayList(); + final List explanations = new ArrayList<>(); + final List details = new ArrayList<>(); StringBuilder currentExplanation; StringBuilder currentDetail; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java index 19d5d16a32..734beb85f8 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java @@ -365,7 +365,7 @@ private static EvalNode createSingletonExprFromCNFRecursive(EvalNode[] evalNode, * @return An array of CNF-formed expressions */ public static EvalNode [] toConjunctiveNormalFormArray(EvalNode expr) { - List list = new ArrayList(); + List list = new ArrayList<>(); toConjunctiveNormalFormArrayRecursive(expr, list); return list.toArray(new EvalNode[list.size()]); } @@ -412,7 +412,7 @@ private static EvalNode createSingletonExprFromDNFRecursive(EvalNode[] evalNode, * @return An array of CNF-formed expressions */ public static EvalNode [] toDisjunctiveNormalFormArray(EvalNode...exprs) { - List list = new ArrayList(); + List list = new ArrayList<>(); for (EvalNode expr : exprs) { toDisjunctiveNormalFormArrayRecursive(expr, list); } @@ -437,7 +437,7 @@ public IdentifiableNameBuilder(Expr expr) { } public String build() { - Stack stack = new Stack(); + Stack stack = new Stack<>(); stack.push(expr); try { this.visit(null, stack, expr); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CaseWhenEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CaseWhenEval.java index 2f40974c7b..331b8e138c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CaseWhenEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CaseWhenEval.java @@ -158,7 +158,7 @@ public void postOrder(EvalNodeVisitor visitor) { @Override public Object clone() throws CloneNotSupportedException { CaseWhenEval caseWhenEval = (CaseWhenEval) super.clone(); - caseWhenEval.whens = new ArrayList(); + caseWhenEval.whens = new ArrayList<>(); for (IfThenEval ifThenEval : whens) { caseWhenEval.whens.add((IfThenEval) ifThenEval.clone()); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java index cf005340ae..fe1c373287 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java @@ -408,7 +408,7 @@ public void visit(EvalNode node) { } public static class AllColumnRefFinder implements EvalNodeVisitor { - private List colList = new ArrayList(); + private List colList = new ArrayList<>(); private FieldEval field = null; @Override diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java index 37b77fd730..63e921efc0 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java @@ -79,7 +79,7 @@ public EvalNode visitUnaryEval(LogicalPlanner.PlanContext context, UnaryEval una } // exceptional func names not to use constant folding - private static final Set NON_CONSTANT_FUNC_NAMES = new HashSet(Arrays.asList("sleep", "random")); + private static final Set NON_CONSTANT_FUNC_NAMES = new HashSet<>(Arrays.asList("sleep", "random")); @Override public EvalNode visitFuncCall(LogicalPlanner.PlanContext context, FunctionEval evalNode, Stack stack) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java index a3b529cce2..292e967c66 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraphContext.java @@ -122,12 +122,12 @@ public void replaceRootVertexes(JoinVertex oldRoot, JoinVertex newRoot) { } public JoinEdge cacheEdge(JoinEdge edge) { - edgeCache.put(new Pair(edge.getLeftVertex(), edge.getRightVertex()), edge); + edgeCache.put(new Pair<>(edge.getLeftVertex(), edge.getRightVertex()), edge); return edge; } public JoinEdge getCachedOrNewJoinEdge(JoinSpec joinSpec, JoinVertex left, JoinVertex right) { - Pair cacheKey = new Pair(left, right); + Pair cacheKey = new Pair<>(left, right); if (edgeCache.containsKey(cacheKey)) { return (JoinEdge) edgeCache.get(cacheKey); } else { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java index fb19f107bb..5ec0f022bf 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java @@ -122,7 +122,7 @@ public Object clone() throws CloneNotSupportedException { } if (subGroupbyPlan != null) { - cloneNode.subGroupbyPlan = new ArrayList(); + cloneNode.subGroupbyPlan = new ArrayList<>(); for (GroupbyNode eachNode: subGroupbyPlan) { GroupbyNode groupbyNode = (GroupbyNode)eachNode.clone(); groupbyNode.setPID(-1); @@ -243,7 +243,7 @@ public PlanString getPlanString() { } public Column[] getFirstStageShuffleKeyColumns() { - List shuffleKeyColumns = new ArrayList(); + List shuffleKeyColumns = new ArrayList<>(); shuffleKeyColumns.add(getOutSchema().getColumn(0)); //distinctseq column if (groupingColumns != null) { for (Column eachColumn: groupingColumns) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinSpec.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinSpec.java index 7643e4b836..abdafc72b5 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinSpec.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinSpec.java @@ -39,7 +39,7 @@ public int compare(EvalNode e1, EvalNode e2) { } private JoinType type = null; - private Set predicates = new TreeSet(new EvalNodeComparator()); + private Set predicates = new TreeSet<>(new EvalNodeComparator()); public JoinSpec() { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java index f26e19064a..e41b7a70f2 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java @@ -177,7 +177,7 @@ public static Column[] getColumns(LogicalPlanner.PlanContext ctx, QualifiedAster private static List resolveAsterisk(LogicalPlanner.PlanContext ctx, QualifiedAsteriskExpr asteriskExpr) throws TajoException { Column[] columns = getColumns(ctx, asteriskExpr); - List newTargetExprs = new ArrayList(columns.length); + List newTargetExprs = new ArrayList<>(columns.length); int i; for (i = 0; i < columns.length; i++) { newTargetExprs.add(new NamedExpr(new ColumnReferenceExpr(columns[i].getQualifier(), columns[i].getSimpleName()))); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java index 408f2d2c38..abfc7a94bc 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java @@ -82,7 +82,7 @@ public LogicalPlan rewrite(LogicalPlanRewriteRuleContext rewriteRuleContext) thr LogicalPlan.QueryBlock topmostBlock = rootBlock; - Stack stack = new Stack(); + Stack stack = new Stack<>(); Context context = new Context(plan); visit(context, plan, topmostBlock, topmostBlock.getRoot(), stack); @@ -406,17 +406,17 @@ static class Context { Set requiredSet; public Context(LogicalPlan plan) { - requiredSet = new LinkedHashSet(); + requiredSet = new LinkedHashSet<>(); targetListMgr = new TargetListManager(plan); } public Context(LogicalPlan plan, Collection requiredSet) { - this.requiredSet = new LinkedHashSet(requiredSet); + this.requiredSet = new LinkedHashSet<>(requiredSet); targetListMgr = new TargetListManager(plan); } public Context(Context upperContext) { - this.requiredSet = new LinkedHashSet(upperContext.requiredSet); + this.requiredSet = new LinkedHashSet<>(upperContext.requiredSet); targetListMgr = upperContext.targetListMgr; } @@ -567,7 +567,7 @@ public LogicalNode visitSort(Context context, LogicalPlan plan, LogicalPlan.Quer LogicalNode child = super.visitSort(newContext, plan, block, node, stack); // it rewrite sortkeys. This rewrite sets right column names and eliminates duplicated sort keys. - List sortSpecs = new ArrayList(); + List sortSpecs = new ArrayList<>(); for (int i = 0; i < keyNames.length; i++) { String sortKey = keyNames[i]; Target target = context.targetListMgr.getTarget(sortKey); @@ -763,7 +763,7 @@ public LogicalNode visitGroupBy(Context context, LogicalPlan plan, LogicalPlan.Q List targets = Lists.newArrayList(); if (groupingKeyNum > 0 && groupingKeyNames != null) { // Restoring grouping key columns - final List groupingColumns = new ArrayList(); + final List groupingColumns = new ArrayList<>(); for (String groupingKey : groupingKeyNames) { Target target = context.targetListMgr.getTarget(groupingKey); @@ -1009,7 +1009,7 @@ static class FilteredIterator implements Iterator { FilteredIterator(Target [] targets, Set requiredReferences) { List filtered = TUtil.newList(); - Map targetSet = new HashMap(); + Map targetSet = new HashMap<>(); for (Target t : targets) { // Only should keep an raw target instead of field reference. if (targetSet.containsKey(t.getCanonicalName())) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 7ba31bc227..d722884b67 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -494,7 +494,7 @@ public static T findTopParentNode(LogicalNode node, Node } private static class LogicalNodeFinder implements LogicalNodeVisitor { - private List list = new ArrayList(); + private List list = new ArrayList<>(); private final NodeType[] tofind; private boolean topmost = false; private boolean finished = false; @@ -532,7 +532,7 @@ public LogicalNode[] getFoundNodeArray() { } private static class ParentNodeFinder implements LogicalNodeVisitor { - private List list = new ArrayList(); + private List list = new ArrayList<>(); private NodeType tofind; public ParentNodeFinder(NodeType type) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java index f826479c22..23ff3dc341 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java @@ -34,7 +34,7 @@ public class ExplainLogicalPlanVisitor extends BasicLogicalPlanVisitor explains = new Stack(); + public Stack explains = new Stack<>(); public void add(int depth, PlanString planString) { maxDepth = Math.max(maxDepth, depth); @@ -69,7 +69,7 @@ public PlanString getPlanString() { } public Context getBlockPlanStrings(@Nullable LogicalPlan plan, LogicalNode node) throws TajoException { - Stack stack = new Stack(); + Stack stack = new Stack<>(); Context explainContext = new Context(); visit(explainContext, plan, null, node, stack); return explainContext; diff --git a/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/HttpDataServerHandler.java b/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/HttpDataServerHandler.java index 472b96709a..68e80931fe 100644 --- a/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/HttpDataServerHandler.java +++ b/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/HttpDataServerHandler.java @@ -47,7 +47,7 @@ public class HttpDataServerHandler extends SimpleChannelInboundHandler retrievers = - new ConcurrentHashMap(); + new ConcurrentHashMap<>(); private String userName; private String appId; @@ -219,7 +219,7 @@ private List splitMaps(List qids) { return null; } - final List ret = new ArrayList(); + final List ret = new ArrayList<>(); for (String qid : qids) { Collections.addAll(ret, qid.split(",")); } diff --git a/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java b/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java index d6f5a9038d..2eaa3d2ff1 100644 --- a/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java +++ b/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/TajoPullServerService.java @@ -105,7 +105,7 @@ public class TajoPullServerService extends AbstractService { public static final String PULLSERVER_SERVICEID = "tajo.pullserver"; private static final Map userRsrc = - new ConcurrentHashMap(); + new ConcurrentHashMap<>(); private String userName; public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY = @@ -360,7 +360,7 @@ protected void initChannel(SocketChannel channel) throws Exception { } - Map processingStatusMap = new ConcurrentHashMap(); + Map processingStatusMap = new ConcurrentHashMap<>(); public void completeFileChunk(FileRegion filePart, String requestUri, @@ -763,7 +763,7 @@ public static List splitMaps(List mapq) { if (null == mapq) { return null; } - final List ret = new ArrayList(); + final List ret = new ArrayList<>(); for (String s : mapq) { Collections.addAll(ret, s.split(",")); } diff --git a/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/retriever/AdvancedDataRetriever.java b/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/retriever/AdvancedDataRetriever.java index fb91094845..59e89047a3 100644 --- a/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/retriever/AdvancedDataRetriever.java +++ b/tajo-pullserver/src/main/java/org/apache/tajo/pullserver/retriever/AdvancedDataRetriever.java @@ -119,7 +119,7 @@ private List splitMaps(List qids) { return null; } - final List ret = new ArrayList(); + final List ret = new ArrayList<>(); for (String qid : qids) { Collections.addAll(ret, qid.split(",")); } diff --git a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/NettyUtils.java b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/NettyUtils.java index 01fd48bcd2..90b337a42f 100644 --- a/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/NettyUtils.java +++ b/tajo-rpc/tajo-rpc-common/src/main/java/org/apache/tajo/rpc/NettyUtils.java @@ -44,7 +44,7 @@ public enum GROUP { } private static final Map eventLoopGroupMap = - new ConcurrentHashMap(); + new ConcurrentHashMap<>(); private NettyUtils(){ } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskDeviceInfo.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskDeviceInfo.java index 23963497c8..a0580564b8 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskDeviceInfo.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskDeviceInfo.java @@ -25,7 +25,7 @@ public class DiskDeviceInfo { private int id; private String name; - private List mountInfos = new ArrayList(); + private List mountInfos = new ArrayList<>(); public DiskDeviceInfo(int id) { this.id = id; diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskUtil.java index 19e08e8066..7629d58465 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskUtil.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/DiskUtil.java @@ -68,7 +68,7 @@ public static List getDiskDeviceInfos() throws IOException { } private static List getUnixDiskDeviceInfos() { - List infos = new ArrayList(); + List infos = new ArrayList<>(); File file = new File(UNIX_DISK_DEVICE_PATH); if(!file.exists()) { @@ -82,7 +82,7 @@ private static List getUnixDiskDeviceInfos() { String line = null; int count = 0; - Set deviceNames = new TreeSet(); + Set deviceNames = new TreeSet<>(); while((line = reader.readLine()) != null) { if(count > 0 && !line.trim().isEmpty()) { String[] tokens = line.trim().split(" +"); @@ -137,7 +137,7 @@ public static List getDefaultDiskDeviceInfos() { DiskDeviceInfo diskDeviceInfo = new DiskDeviceInfo(0); diskDeviceInfo.setName("default"); - List infos = new ArrayList(); + List infos = new ArrayList<>(); infos.add(diskDeviceInfo); @@ -146,7 +146,7 @@ public static List getDefaultDiskDeviceInfos() { private static void setDeviceMountInfo(List deviceInfos) throws IOException { - Map deviceMap = new HashMap(); + Map deviceMap = new HashMap<>(); for(DiskDeviceInfo eachDevice: deviceInfos) { deviceMap.put(eachDevice.getName(), eachDevice); } diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java index b3b1edd6ea..27ff5895b5 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/MergeScanner.java @@ -64,7 +64,7 @@ public MergeScanner(Configuration conf, Schema schema, TableMeta meta, List(); + this.fragments = new ArrayList<>(); long numBytes = 0; for (Fragment eachFileFragment: rawFragmentList) { diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java index 2e00bc9d6e..9716744c65 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/OldStorageManager.java @@ -48,12 +48,12 @@ public class OldStorageManager { * Cache of scanner handlers for each storage type. */ protected static final Map> SCANNER_HANDLER_CACHE - = new ConcurrentHashMap>(); + = new ConcurrentHashMap<>(); /** * Cache of appender handlers for each storage type. */ protected static final Map> APPENDER_HANDLER_CACHE - = new ConcurrentHashMap>(); + = new ConcurrentHashMap<>(); private static final Class[] DEFAULT_SCANNER_PARAMS = { Configuration.class, Schema.class, diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/compress/CodecPool.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/compress/CodecPool.java index baeda8cace..6545e4e223 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/compress/CodecPool.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/compress/CodecPool.java @@ -43,14 +43,14 @@ public final class CodecPool { * construction/destruction of (possibly native) decompression codecs. */ private static final Map, List> COMPRESSOR_POOL = - new HashMap, List>(); + new HashMap<>(); /** * A global decompressor pool used to save the expensive * construction/destruction of (possibly native) decompression codecs. */ private static final Map, List> DECOMPRESSOR_POOL = - new HashMap, List>(); + new HashMap<>(); private static T borrow(Map, List> pool, Class codecClass) { diff --git a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestTupleComparator.java b/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestTupleComparator.java index 9eec96fe66..521abf1aec 100644 --- a/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestTupleComparator.java +++ b/tajo-storage/tajo-storage-common/src/test/java/org/apache/tajo/storage/TestTupleComparator.java @@ -93,7 +93,7 @@ public void testNullFirst() throws Exception { tuple3.put(0, new Int4Datum(3)); tuple3.put(1, new TextDatum("333")); - List tupleList = new ArrayList(); + List tupleList = new ArrayList<>(); tupleList.add(tuple1); tupleList.add(nullTuple); tupleList.add(tuple3); @@ -133,7 +133,7 @@ public void testNullLast() throws Exception { tuple3.put(0, new Int4Datum(3)); tuple3.put(1, new TextDatum("333")); - List tupleList = new ArrayList(); + List tupleList = new ArrayList<>(); tupleList.add(tuple1); tupleList.add(nullTuple); tupleList.add(tuple3); diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java index af78d13a1b..2e7db7f921 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/AbstractHBaseAppender.java @@ -102,7 +102,7 @@ public void init() throws IOException { mappingColumnFamilies = columnMapping.getMappingColumns(); isRowKeyMappings = columnMapping.getIsRowKeyMappings(); - List rowkeyColumnIndexList = new ArrayList(); + List rowkeyColumnIndexList = new ArrayList<>(); for (int i = 0; i < isRowKeyMappings.length; i++) { if (isRowKeyMappings[i]) { rowkeyColumnIndexList.add(i); @@ -122,7 +122,7 @@ public void init() throws IOException { columnKeyValueDataIndexes = new int[isColumnKeys.length]; int index = 0; int numKeyValues = 0; - Map cfNameIndexMap = new HashMap(); + Map cfNameIndexMap = new HashMap<>(); for (int i = 0; i < isColumnKeys.length; i++) { if (isRowKeyMappings[i]) { continue; diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java index 7df7e9c3e0..be05d7ce07 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/ColumnMapping.java @@ -170,7 +170,7 @@ public void init() throws MissingTablePropertyException, InvalidTablePropertyExc } public List getColumnFamilyNames() { - List cfNames = new ArrayList(); + List cfNames = new ArrayList<>(); for (byte[][] eachCfName: mappingColumns) { if (eachCfName != null && eachCfName.length > 0 && eachCfName[0] != null) { diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java index 0064be4cbc..91840d4a93 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java @@ -79,7 +79,7 @@ public class HBaseTablespace extends Tablespace { private final static SortedInsertRewriter REWRITE_RULE = new SortedInsertRewriter(); - private Map connMap = new HashMap(); + private Map connMap = new HashMap<>(); public HBaseTablespace(String spaceName, URI uri, JSONObject config) { super(spaceName, uri, config); @@ -176,7 +176,7 @@ private void createTable(URI uri, TableMeta tableMeta, Schema schema, "External table should be a existed table."); } HTableDescriptor hTableDescriptor = hAdmin.getTableDescriptor(hTableName); - Set tableColumnFamilies = new HashSet(); + Set tableColumnFamilies = new HashSet<>(); for (HColumnDescriptor eachColumn : hTableDescriptor.getColumnFamilies()) { tableColumnFamilies.add(eachColumn.getNameAsString()); } @@ -278,7 +278,7 @@ private byte[][] getSplitKeys(TajoConf conf, String hbaseTableName, Schema schem hbaseTableName); } - SortedSet splitKeySet = new TreeSet(); + SortedSet splitKeySet = new TreeSet<>(); BufferedReader reader = null; try { reader = new BufferedReader(new InputStreamReader(fs.open(path))); @@ -437,7 +437,7 @@ public List getSplits(String inputSourceId, if (null == regLoc) { throw new IOException("Expecting at least one region."); } - List fragments = new ArrayList(1); + List fragments = new ArrayList<>(1); Fragment fragment = new HBaseFragment( tableDesc.getUri(), inputSourceId, htable.getName().getNameAsString(), @@ -453,8 +453,8 @@ public List getSplits(String inputSourceId, if (indexPredications != null && !indexPredications.isEmpty()) { // indexPredications is Disjunctive set - startRows = new ArrayList(); - stopRows = new ArrayList(); + startRows = new ArrayList<>(); + stopRows = new ArrayList<>(); for (IndexPredication indexPredication: indexPredications) { byte[] startRow; byte[] stopRow; @@ -477,10 +477,10 @@ public List getSplits(String inputSourceId, } hAdmin = new HBaseAdmin(hbaseConf); - Map serverLoadMap = new HashMap(); + Map serverLoadMap = new HashMap<>(); // region startkey -> HBaseFragment - Map fragmentMap = new HashMap(); + Map fragmentMap = new HashMap<>(); for (int i = 0; i < keys.getFirst().length; i++) { HRegionLocation location = htable.getRegionLocation(keys.getFirst()[i], false); @@ -549,7 +549,7 @@ public List getSplits(String inputSourceId, } } - List fragments = new ArrayList(fragmentMap.values()); + List fragments = new ArrayList<>(fragmentMap.values()); Collections.sort(fragments); if (!fragments.isEmpty()) { fragments.get(fragments.size() - 1).setLast(true); @@ -622,7 +622,7 @@ static class HConnectionKey { private String username; HConnectionKey(Configuration conf) { - Map m = new HashMap(); + Map m = new HashMap<>(); if (conf != null) { for (String property : CONNECTION_PROPERTIES) { String value = conf.get(property); @@ -712,7 +712,7 @@ public List getIndexPredications(ColumnMapping columnMapping, @Nullable EvalNode filterCondition) throws IOException, MissingTablePropertyException, InvalidTablePropertyException { - List indexPredications = new ArrayList(); + List indexPredications = new ArrayList<>(); Column[] indexableColumns = getIndexableColumns(tableDesc); if (indexableColumns != null && indexableColumns.length == 1) { // Currently supports only single index column. @@ -735,7 +735,7 @@ public List getIndexPredications(ColumnMapping columnMapping, public List> findIndexablePredicateSet(@Nullable EvalNode qual, Column[] indexableColumns) throws IOException { - List> indexablePredicateList = new ArrayList>(); + List> indexablePredicateList = new ArrayList<>(); // if a query statement has a search condition, try to find indexable predicates if (indexableColumns != null && qual != null) { @@ -885,7 +885,7 @@ public Pair getIndexablePredicateValue(ColumnMapping columnMapping new String(new char[]{columnMapping.getRowKeyDelimiter(), Character.MAX_VALUE})); } if (startDatum != null || endDatum != null) { - return new Pair(startDatum, endDatum); + return new Pair<>(startDatum, endDatum); } else { return null; } @@ -955,7 +955,7 @@ public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc if (endKeys.length == 1) { return new TupleRange[]{dataRange}; } - List tupleRanges = new ArrayList(endKeys.length); + List tupleRanges = new ArrayList<>(endKeys.length); TupleComparator comparator = new BaseTupleComparator(inputSchema, sortSpecs); Tuple previousTuple = dataRange.getStart(); diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java index 36678e41f5..42f25cc915 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HFileAppender.java @@ -87,7 +87,7 @@ public void init() throws IOException { long totalNumBytes = 0; ImmutableBytesWritable keyWritable = new ImmutableBytesWritable(); boolean first = true; - TreeSet kvSet = new TreeSet(KeyValue.COMPARATOR); + TreeSet kvSet = new TreeSet<>(KeyValue.COMPARATOR); @Override diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/SortedInsertRewriter.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/SortedInsertRewriter.java index a99d4d08ae..5c418df1b0 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/SortedInsertRewriter.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/SortedInsertRewriter.java @@ -58,7 +58,7 @@ public boolean isEligible(LogicalPlanRewriteRuleContext context) { public static Column[] getIndexColumns(Schema tableSchema, KeyValueSet tableProperty) throws IOException, TajoException { - List indexColumns = new ArrayList(); + List indexColumns = new ArrayList<>(); ColumnMapping columnMapping = new ColumnMapping(tableSchema, tableProperty); diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java index 1b0952ea8e..6ba2029279 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java @@ -204,7 +204,7 @@ private FileFragment[] split(String tableName, Path tablePath, long size) FileSystem fs = tablePath.getFileSystem(conf); long defaultBlockSize = size; - List listTablets = new ArrayList(); + List listTablets = new ArrayList<>(); FileFragment tablet; FileStatus[] fileLists = fs.listStatus(tablePath); @@ -236,7 +236,7 @@ public static FileFragment[] splitNG(Configuration conf, String tableName, Table FileSystem fs = tablePath.getFileSystem(conf); long defaultBlockSize = size; - List listTablets = new ArrayList(); + List listTablets = new ArrayList<>(); FileFragment tablet; FileStatus[] fileLists = fs.listStatus(tablePath); @@ -324,16 +324,16 @@ public boolean accept(Path path) { * @throws IOException if zero items. */ protected List listStatus(Path... dirs) throws IOException { - List result = new ArrayList(); + List result = new ArrayList<>(); if (dirs.length == 0) { throw new IOException("No input paths specified in job"); } - List errors = new ArrayList(); + List errors = new ArrayList<>(); // creates a MultiPathFilter with the hiddenFileFilter and the // user provided one (if any). - List filters = new ArrayList(); + List filters = new ArrayList<>(); filters.add(hiddenFileFilter); PathFilter inputFilter = new MultiPathFilter(filters); @@ -426,7 +426,7 @@ protected FileFragment makeSplit(String fragmentId, Path file, BlockLocation blo protected FileFragment makeNonSplit(String fragmentId, Path file, long start, long length, BlockLocation[] blkLocations) throws IOException { - Map hostsBlockMap = new HashMap(); + Map hostsBlockMap = new HashMap<>(); for (BlockLocation blockLocation : blkLocations) { for (String host : blockLocation.getHosts()) { if (hostsBlockMap.containsKey(host)) { @@ -437,7 +437,7 @@ protected FileFragment makeNonSplit(String fragmentId, Path file, long start, lo } } - List> entries = new ArrayList>(hostsBlockMap.entrySet()); + List> entries = new ArrayList<>(hostsBlockMap.entrySet()); Collections.sort(entries, new Comparator>() { @Override diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java index 4c772c9c90..a1eead25c8 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppender.java @@ -45,7 +45,7 @@ public class HashShuffleAppender implements Appender { private Map>>> taskTupleIndexes; //page start offset, length - private List> pages = new ArrayList>(); + private List> pages = new ArrayList<>(); private Pair currentPage; @@ -69,7 +69,7 @@ public HashShuffleAppender(ExecutionBlockId ebId, int partId, int pageSize, File @Override public void init() throws IOException { currentPage = new Pair(0L, 0); - taskTupleIndexes = new HashMap>>>(); + taskTupleIndexes = new HashMap<>(); rowNumInPage = 0; } @@ -98,7 +98,7 @@ public int addTuples(TaskAttemptId taskId, List tuples) throws IOExceptio int nextRowNum = rowNumInPage + tuples.size(); List>> taskIndexes = taskTupleIndexes.get(taskId); if (taskIndexes == null) { - taskIndexes = new ArrayList>>(); + taskIndexes = new ArrayList<>(); taskTupleIndexes.put(taskId, taskIndexes); } taskIndexes.add( @@ -194,7 +194,7 @@ public Map>>> getTaskTuple } public List>> getMergedTupleIndexes() { - List>> merged = new ArrayList>>(); + List>> merged = new ArrayList<>(); for (List>> eachFailureIndex: taskTupleIndexes.values()) { merged.addAll(eachFailureIndex); diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java index b118044623..4297e4d1ba 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java @@ -43,7 +43,7 @@ public class HashShuffleAppenderManager { private static final Log LOG = LogFactory.getLog(HashShuffleAppenderManager.class); private Map> appenderMap = - new ConcurrentHashMap>(); + new ConcurrentHashMap<>(); private TajoConf systemConf; private FileSystem defaultFS; private FileSystem localFS; @@ -68,7 +68,7 @@ public HashShuffleAppender getAppender(TajoConf tajoConf, ExecutionBlockId ebId, Map partitionAppenderMap = appenderMap.get(ebId); if (partitionAppenderMap == null) { - partitionAppenderMap = new ConcurrentHashMap(); + partitionAppenderMap = new ConcurrentHashMap<>(); appenderMap.put(ebId, partitionAppenderMap); } @@ -137,7 +137,7 @@ public List close(ExecutionBlockId ebId) throws IOExcep } // Send Intermediate data to QueryMaster. - List intermediateEntries = new ArrayList(); + List intermediateEntries = new ArrayList<>(); for (PartitionAppenderMeta eachMeta : partitionAppenderMap.values()) { try { eachMeta.appender.close(); @@ -180,7 +180,7 @@ public static class HashShuffleIntermediate { private Collection>> failureTskTupleIndexes; //[] - private List> pages = new ArrayList>(); + private List> pages = new ArrayList<>(); public HashShuffleIntermediate(int partId, long volume, List> pages, diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java index 0c67320966..ff0e8c04ca 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroAppender.java @@ -80,8 +80,8 @@ public void init() throws IOException { avroFields = avroSchema.getFields(); DatumWriter datumWriter = - new GenericDatumWriter(avroSchema); - dataFileWriter = new DataFileWriter(datumWriter); + new GenericDatumWriter<>(avroSchema); + dataFileWriter = new DataFileWriter<>(datumWriter); dataFileWriter.create(avroSchema, outputStream); if (enabledStats) { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java index 5946e08327..afa27018d8 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java @@ -85,9 +85,9 @@ public void init() throws IOException { avroSchema = AvroUtil.getAvroSchema(meta, conf); avroFields = avroSchema.getFields(); - DatumReader datumReader = new GenericDatumReader(avroSchema); + DatumReader datumReader = new GenericDatumReader<>(avroSchema); SeekableInput input = new FsInput(fragment.getPath(), conf); - dataFileReader = new DataFileReader(input, datumReader); + dataFileReader = new DataFileReader<>(input, datumReader); super.init(); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java index 5f94aae554..6ac5fc3259 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/fragment/FileFragment.java @@ -217,7 +217,7 @@ public FragmentProto getProto() { builder.setLength(this.length); builder.setPath(this.uri.toString()); if(diskIds != null) { - List idList = new ArrayList(); + List idList = new ArrayList<>(); for(int eachId: diskIds) { idList.add(eachId); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java index ba81c3ea56..3affd5088a 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java @@ -256,14 +256,14 @@ private class KeyOffsetCollector { private TreeMap> map; public KeyOffsetCollector(TupleComparator comparator) { - map = new TreeMap>(comparator); + map = new TreeMap<>(comparator); } public void put(final Tuple key, final long offset) { if (map.containsKey(key)) { map.get(key).add(offset); } else { - LinkedList list = new LinkedList(); + LinkedList list = new LinkedList<>(); list.add(offset); map.put(key, list); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCScanner.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCScanner.java index 8a9d623e88..b8b75eda8e 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCScanner.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/ORCScanner.java @@ -148,7 +148,7 @@ public void init() throws IOException { vectors[i] = createOrcVector(targetColInfo[i].type); } - Set columnSet = new HashSet(); + Set columnSet = new HashSet<>(); for (ColumnInfo colInfo: targetColInfo) { columnSet.add(colInfo.id); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/objectinspector/TajoStructObjectInspector.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/objectinspector/TajoStructObjectInspector.java index a8b4b4945b..f28989b5c8 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/objectinspector/TajoStructObjectInspector.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/orc/objectinspector/TajoStructObjectInspector.java @@ -69,7 +69,7 @@ public String getFieldComment() { } TajoStructObjectInspector(Schema schema) { - structFields = new ArrayList(schema.size()); + structFields = new ArrayList<>(schema.size()); for (Column c: schema.getRootColumns()) { try { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java index 555b623f1c..e4f4730413 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoSchemaConverter.java @@ -54,7 +54,7 @@ public Schema convert(MessageType parquetSchema) { } private Schema convertFields(List parquetFields) { - List columns = new ArrayList(); + List columns = new ArrayList<>(); for (int i = 0; i < parquetFields.size(); ++i) { Type fieldType = parquetFields.get(i); if (fieldType.isRepetition(Type.Repetition.REPEATED)) { @@ -141,7 +141,7 @@ private Column convertComplexField(final Type fieldType) { * @return The resulting Parquet schema. */ public MessageType convert(Schema tajoSchema) { - List types = new ArrayList(); + List types = new ArrayList<>(); for (int i = 0; i < tajoSchema.size(); ++i) { Column column = tajoSchema.getColumn(i); if (column.getDataType().getType() == TajoDataTypes.Type.NULL_TYPE) { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java index de2a1e3613..acb901580e 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/parquet/TajoWriteSupport.java @@ -62,7 +62,7 @@ public TajoWriteSupport(Schema tajoSchema) { */ @Override public WriteContext init(Configuration configuration) { - Map extraMetaData = new HashMap(); + Map extraMetaData = new HashMap<>(); return new WriteContext(rootSchema, extraMetaData); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/ColumnProjectionUtils.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/ColumnProjectionUtils.java index 352776f366..fc106f2739 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/ColumnProjectionUtils.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/ColumnProjectionUtils.java @@ -89,11 +89,11 @@ private static String toReadColumnIDString(ArrayList ids) { */ public static ArrayList getReadColumnIDs(Configuration conf) { if (conf == null) { - return new ArrayList(0); + return new ArrayList<>(0); } String skips = conf.get(READ_COLUMN_IDS_CONF_STR, ""); String[] list = StringUtils.split(skips); - ArrayList result = new ArrayList(list.length); + ArrayList result = new ArrayList<>(list.length); for (String element : list) { // it may contain duplicates, remove duplicates Integer toAdd = Integer.parseInt(element); diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java index bc1d7ba9ba..b5155d4169 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java @@ -66,7 +66,7 @@ public class DelimitedTextFile { /** it caches line serde classes. */ private static final Map> serdeClassCache = - new ConcurrentHashMap>(); + new ConcurrentHashMap<>(); /** * By default, DelimitedTextFileScanner uses CSVLineSerder. If a table property 'text.serde.class' is given, diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/FileOrcDataSource.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/FileOrcDataSource.java index dcc134705b..de7c802fcd 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/FileOrcDataSource.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/FileOrcDataSource.java @@ -109,7 +109,7 @@ public Map readFully(Map diskRanges) Iterable mergedRanges = mergeAdjacentDiskRanges(diskRanges.values(), maxMergeDistance); // read ranges - Map buffers = new LinkedHashMap(); + Map buffers = new LinkedHashMap<>(); for (DiskRange mergedRange : mergedRanges) { // read full range in one request byte[] buffer = new byte[mergedRange.getLength()]; diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/HdfsOrcDataSource.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/HdfsOrcDataSource.java index 73ea47538d..da12461a53 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/HdfsOrcDataSource.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/HdfsOrcDataSource.java @@ -106,7 +106,7 @@ public Map readFully(Map diskRanges) Iterable mergedRanges = mergeAdjacentDiskRanges(diskRanges.values(), maxMergeDistance); // read ranges - Map buffers = new LinkedHashMap(); + Map buffers = new LinkedHashMap<>(); for (DiskRange mergedRange : mergedRanges) { // read full range in one request byte[] buffer = new byte[mergedRange.getLength()]; diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/MemoryManager.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/MemoryManager.java index 8cd40f73e9..79af80fbb6 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/MemoryManager.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/MemoryManager.java @@ -52,7 +52,7 @@ class MemoryManager { private static final int ROWS_BETWEEN_CHECKS = 5000; private final long totalMemoryPool; private final Map writerList = - new HashMap(); + new HashMap<>(); private long totalAllocation = 0; private double currentScale = 1; private int rowsAddedSinceCheck = 0; diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/OrcUtils.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/OrcUtils.java index 847c10cf69..28ddd02787 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/OrcUtils.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/OrcUtils.java @@ -91,7 +91,7 @@ private static Map> getColumnSpan(String allColumns, // schema: struct> // column span map for the above struct will be // a => [1,1], b => [2,2], c => [3,5] - Map> columnSpanMap = new HashMap>(); + Map> columnSpanMap = new HashMap<>(); if (allColumns != null) { String[] columns = allColumns.split(","); int startIdx = 0; diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java index e520082b9a..1e5e720d2d 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/WriterImpl.java @@ -99,7 +99,7 @@ public class WriterImpl implements Writer, MemoryManager.Callback { private final float paddingTolerance; // the streams that make up the current stripe private final Map streams = - new TreeMap(); + new TreeMap<>(); private FSDataOutputStream rawWriter = null; // the compressed metadata information outStream @@ -114,9 +114,9 @@ public class WriterImpl implements Writer, MemoryManager.Callback { private int rowsInIndex = 0; private int stripesAtLastFlush = -1; private final List stripes = - new ArrayList(); + new ArrayList<>(); private final Map userMetadata = - new TreeMap(); + new TreeMap<>(); private final TreeWriter treeWriter; private final boolean buildIndex; private final MemoryManager memoryManager; @@ -343,7 +343,7 @@ public boolean checkMemory(double newScale) throws IOException { */ private class BufferedStream implements OutStream.OutputReceiver { private final OutStream outStream; - private final List output = new ArrayList(); + private final List output = new ArrayList<>(); BufferedStream(String name, int bufferSize, CompressionCodec codec) throws IOException { @@ -1150,9 +1150,9 @@ private static class StringTreeWriter extends TreeWriter { private final PositionedOutputStream directStreamOutput; private final IntegerWriter directLengthOutput; private final List savedRowIndex = - new ArrayList(); + new ArrayList<>(); private final boolean buildIndex; - private final List rowIndexValueCount = new ArrayList(); + private final List rowIndexValueCount = new ArrayList<>(); // If the number of keys in a dictionary is greater than this fraction of //the total number of non-null rows, turn off dictionary encoding private final float dictionaryKeySizeThreshold; diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/CodecFactory.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/CodecFactory.java index 4ba47c1379..fcaf2f33fd 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/CodecFactory.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/CodecFactory.java @@ -123,9 +123,9 @@ public CompressionCodecName getCodecName() { } - private final Map compressors = new HashMap(); - private final Map decompressors = new HashMap(); - private final Map codecByName = new HashMap(); + private final Map compressors = new HashMap<>(); + private final Map decompressors = new HashMap<>(); + private final Map codecByName = new HashMap<>(); private final Configuration configuration; public CodecFactory(Configuration configuration) { diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ColumnChunkPageWriteStore.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ColumnChunkPageWriteStore.java index 91d4748adc..6507b80df5 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ColumnChunkPageWriteStore.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ColumnChunkPageWriteStore.java @@ -56,7 +56,7 @@ private static final class ColumnChunkPageWriter implements PageWriter { private long totalValueCount; private int pageCount; - private Set encodings = new HashSet(); + private Set encodings = new HashSet<>(); private Statistics totalStatistics; @@ -138,7 +138,7 @@ public void writeToFileWriter(ParquetFileWriter writer) throws IOException { writer.writeDictionaryPage(dictionaryPage); encodings.add(dictionaryPage.getEncoding()); } - writer.writeDataPages(BytesInput.from(buf), uncompressedLength, compressedLength, totalStatistics, new ArrayList(encodings)); + writer.writeDataPages(BytesInput.from(buf), uncompressedLength, compressedLength, totalStatistics, new ArrayList<>(encodings)); writer.endColumn(); if (DEBUG) { LOG.debug( @@ -176,7 +176,7 @@ public String memUsageString(String prefix) { } } - private final Map writers = new HashMap(); + private final Map writers = new HashMap<>(); private final MessageType schema; private final BytesCompressor compressor; private final int initialSize; diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetFileWriter.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetFileWriter.java index ac1c421404..7a32329bb1 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetFileWriter.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/parquet/ParquetFileWriter.java @@ -64,7 +64,7 @@ public class ParquetFileWriter { private BlockMetaData currentBlock; private ColumnChunkMetaData currentColumn; private long currentRecordCount; - private List blocks = new ArrayList(); + private List blocks = new ArrayList<>(); private long uncompressedLength; private long compressedLength; private Set currentEncodings; @@ -181,7 +181,7 @@ public void startColumn(ColumnDescriptor descriptor, CompressionCodecName compressionCodecName) throws IOException { state = state.startColumn(); if (DEBUG) LOG.debug(out.getPos() + ": start column: " + descriptor + " count=" + valueCount); - currentEncodings = new HashSet(); + currentEncodings = new HashSet<>(); currentChunkPath = ColumnPath.get(descriptor.getPath()); currentChunkType = descriptor.getType(); currentChunkCodec = compressionCodecName; @@ -399,7 +399,7 @@ public static void writeMetadataFile(Configuration configuration, Path outputPat private static ParquetMetadata mergeFooters(Path root, List