From 7b3af51a3d6ec538a55c8b0172205a38b87f4e22 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Fri, 18 Aug 2017 20:13:21 +0300 Subject: [PATCH 01/40] IGNITE-6024: support for server-side dml --- .../ignite/codegen/MessageCodeGenerator.java | 6 +- .../query/h2/DmlStatementsProcessor.java | 151 +++- .../processors/query/h2/IgniteH2Indexing.java | 79 +- .../processors/query/h2/dml/UpdatePlan.java | 20 +- .../query/h2/dml/UpdatePlanBuilder.java | 62 +- .../h2/twostep/DistributedUpdateRun.java | 104 +++ .../h2/twostep/GridMapQueryExecutor.java | 64 ++ .../h2/twostep/GridReduceQueryExecutor.java | 165 ++++- .../h2/twostep/msg/GridH2DmlRequest.java | 697 ++++++++++++++++++ .../h2/twostep/msg/GridH2DmlResponse.java | 226 ++++++ .../msg/GridH2ValueMessageFactory.java | 6 + 11 files changed, 1545 insertions(+), 35 deletions(-) create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java diff --git a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java index 99ec08ad290b2..b5499b47a3812 100644 --- a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java +++ b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java @@ -46,6 +46,8 @@ import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx; +import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlRequest; +import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlResponse; import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteUuid; @@ -161,7 +163,7 @@ private static MessageCollectionItemType typeEnum(Class cls) { * @throws Exception In case of error. */ public static void main(String[] args) throws Exception { - String srcDir = DFLT_SRC_DIR; + String srcDir = INDEXING_SRC_DIR;//DFLT_SRC_DIR; if (args != null && args.length > 0) srcDir = args[0]; @@ -235,6 +237,8 @@ public static void main(String[] args) throws Exception { // gen.generateAndWrite(GridH2RowMessage.class); // gen.generateAndWrite(GridCacheVersion.class); // gen.generateAndWrite(GridCacheVersionEx.class); + gen.generateAndWrite(GridH2DmlRequest.class); + gen.generateAndWrite(GridH2DmlResponse.class); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 0ff9cfef6b60a..b7b90ddd007fb 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.processors.query.h2; import java.lang.reflect.Array; +import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Time; @@ -54,6 +55,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.QueryCursorImpl; +import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery; import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator; import org.apache.ignite.internal.processors.query.GridQueryCancel; @@ -68,7 +70,9 @@ import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlan; import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlanBuilder; import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; +import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser; +import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter; import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap; import org.apache.ignite.internal.util.lang.IgniteSingletonIterator; import org.apache.ignite.internal.util.typedef.F; @@ -83,6 +87,7 @@ import org.h2.command.dml.Insert; import org.h2.command.dml.Merge; import org.h2.command.dml.Update; +import org.h2.jdbc.JdbcPreparedStatement; import org.h2.table.Column; import org.h2.util.DateTimeUtils; import org.h2.util.LocalDateTimeUtils; @@ -163,7 +168,7 @@ private UpdateResult updateSqlFields(String schemaName, PreparedStatement stmt, long items = 0; - UpdatePlan plan = getPlanForStatement(schemaName, stmt, null); + UpdatePlan plan = getPlanForStatement(schemaName, stmt, fieldsQry, loc, null); GridCacheContext cctx = plan.tbl.rowDescriptor().context(); @@ -210,6 +215,33 @@ else if (items == 0L) return new UpdateResult(items, errKeys); } + private boolean checkDistributed(String schemaName, UpdatePlan p, SqlFieldsQuery fieldsQry) { + Connection c = idx.connectionForSchema(schemaName); + + try { + PreparedStatement stmt = c.prepareStatement(p.selectQry); + + idx.bindParameters(stmt, F.asList(fieldsQry.getArgs())); + + GridCacheTwoStepQuery qry = GridSqlQuerySplitter.split((JdbcPreparedStatement)stmt, + fieldsQry.getArgs(), + fieldsQry.isCollocated(), + fieldsQry.isDistributedJoins(), + fieldsQry.isEnforceJoinOrder(), idx); + + // TODO: find a way to set aside sub-queries + p.distributed = !qry.isReplicatedOnly() && qry.mapQueries().size() == 1 && qry.skipMergeTable(); + + if (p.distributed) + p.cacheIds = idx.collectCacheIds(CU.cacheId(p.tbl.cacheName()), qry); + } + catch (SQLException | IgniteCheckedException e) { + throw new IgniteException(e); + } + + return p.distributed; + } + /** * @param schemaName Schema. * @param stmt Prepared statement. @@ -354,7 +386,7 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo Integer errKeysPos = null; - UpdatePlan plan = getPlanForStatement(schemaName, prepStmt, errKeysPos); + UpdatePlan plan = getPlanForStatement(schemaName, prepStmt, fieldsQry, loc, errKeysPos); if (plan.fastUpdateArgs != null) { assert F.isEmpty(failedKeys) && errKeysPos == null; @@ -362,6 +394,9 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo return doFastUpdate(plan, fieldsQry.getArgs()); } + if (plan.distributed) + return doDistributedUpdate(schemaName, fieldsQry, plan, cancel); + assert !F.isEmpty(plan.selectQry); QueryCursorImpl> cur; @@ -417,6 +452,47 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo } } + /** */ + private UpdateResult executeLocalDmlSelect(GridCacheContext cctx, String schemaName, UpdatePlan plan, + int pageSize, boolean isEnforceJoinOrder, int timeoutMillis, IndexingQueryFilter filters, + GridQueryCancel cancel, Object[] args) throws IgniteCheckedException { + + // TODO: similar code in executeUpdateStatement(), need to refactor + + final GridQueryFieldsResult res = idx.queryLocalSqlFields(schemaName, plan.selectQry, + F.asList(args), filters, isEnforceJoinOrder, timeoutMillis, cancel); + + QueryCursorImpl> cur = new QueryCursorImpl<>(new Iterable>() { + @Override public Iterator> iterator() { + try { + return new GridQueryCacheObjectsIterator(res.iterator(), idx.objectContext(), true); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + }, cancel); + + + switch (plan.mode) { + case MERGE: + return new UpdateResult(doMerge(plan, cur, pageSize), X.EMPTY_OBJECT_ARRAY); + + case INSERT: + return new UpdateResult(doInsert(plan, cur, pageSize), X.EMPTY_OBJECT_ARRAY); + + case UPDATE: + return doUpdate(plan, cur, pageSize); + + case DELETE: + return doDelete(cctx, cur, pageSize); + + default: + throw new IgniteSQLException("Unexpected DML operation [mode=" + plan.mode + ']', + IgniteQueryErrorCode.UNEXPECTED_OPERATION); + } + } + /** * Generate SELECT statements to retrieve data for modifications from and find fast UPDATE or DELETE args, * if available. @@ -426,8 +502,8 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo * @return Update plan. */ @SuppressWarnings({"unchecked", "ConstantConditions"}) - private UpdatePlan getPlanForStatement(String schema, PreparedStatement prepStmt, @Nullable Integer errKeysPos) - throws IgniteCheckedException { + private UpdatePlan getPlanForStatement(String schema, PreparedStatement prepStmt, SqlFieldsQuery fieldsQry, + boolean loc, @Nullable Integer errKeysPos) throws IgniteCheckedException { Prepared p = GridSqlQueryParser.prepared(prepStmt); H2DmlPlanKey planKey = new H2DmlPlanKey(schema, p.getSQL()); @@ -439,6 +515,9 @@ private UpdatePlan getPlanForStatement(String schema, PreparedStatement prepStmt res = UpdatePlanBuilder.planForStatement(p, errKeysPos); + if (!loc && !F.isEmpty(res.selectQry)) + checkDistributed(schema, res, fieldsQry); + // Don't cache re-runs if (errKeysPos == null) return U.firstNotNull(planCache.putIfAbsent(planKey, res), res); @@ -485,6 +564,19 @@ private static UpdateResult doFastUpdate(UpdatePlan plan, Object[] args) throws } } + /** */ + private UpdateResult doDistributedUpdate(String schemaName, SqlFieldsQuery fieldsQry, UpdatePlan plan, + GridQueryCancel cancel) throws IgniteCheckedException { + + if (cancel == null) + cancel = new GridQueryCancel(); + + long cnt = idx.runDistributedUpdate(schemaName, fieldsQry, plan.cacheIds, (byte)plan.mode.ordinal(), + plan.tbl.getName(), plan.colNames, plan.selectQry, cancel); + + return new UpdateResult(cnt, null); + } + /** * Perform DELETE operation on top of results of SELECT. * @param cctx Cache context. @@ -572,7 +664,7 @@ private UpdateResult doUpdate(UpdatePlan plan, Iterable> cursor, int pag GridQueryProperty prop = plan.tbl.rowDescriptor().type().property(plan.colNames[i]); - assert prop != null; + assert prop != null : "Unknown property: " + plan.colNames[i]; newColVals.put(plan.colNames[i], convert(row.get(i + 2), desc, prop.type(), plan.colTypes[i])); } @@ -955,6 +1047,55 @@ private static PageProcessingResult processPage(GridCacheContext cctx, return new IgniteBiTuple<>(key, val); } + /** */ + long mapDistributedUpdate(byte mode, String schemaName, String targetTable, + String[] colNames, String qry, Object[] params, int pageSize, int timeoutMillis, + IndexingQueryFilter filter) throws IgniteCheckedException { + GridH2Table tgtTbl = idx.dataTable(schemaName, targetTable); //TODO: use QueryTable instead + + UpdateMode[] modes = UpdateMode.values(); + + assert mode >= 0 && mode < modes.length; + + UpdateMode updateMode = modes[mode]; + + GridCacheContext cctx = tgtTbl.rowDescriptor().context(); + + UpdatePlan plan = UpdatePlanBuilder.planFromMessage(cctx, updateMode, tgtTbl, tgtTbl.rowDescriptor(), + colNames, qry); + + boolean isEnforceJoinOrder = false; // TODO fix this. + GridQueryCancel cancel = new GridQueryCancel(); + + CacheOperationContext opCtx = cctx.operationContextPerCall(); + + // Force keepBinary for operation context to avoid binary deserialization inside entry processor + if (cctx.binaryMarshaller()) { + CacheOperationContext newOpCtx = null; + + if (opCtx == null) + // Mimics behavior of GridCacheAdapter#keepBinary and GridCacheProxyImpl#keepBinary + newOpCtx = new CacheOperationContext(false, null, true, null, false, null, false); + else if (!opCtx.isKeepBinary()) + newOpCtx = opCtx.keepBinary(); + + if (newOpCtx != null) + cctx.operationContextPerCall(newOpCtx); + } + + UpdateResult res; + + try { + res = executeLocalDmlSelect(cctx, schemaName, plan, pageSize, isEnforceJoinOrder, + timeoutMillis, filter, cancel, params); + } + finally { + cctx.operationContextPerCall(opCtx); + } + + return res.cnt; + } + /** */ private final static class InsertEntryProcessor implements EntryProcessor { /** Value to set. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 6896f18d6a175..39b72398d2268 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1192,6 +1192,22 @@ private Iterable> runQueryTwoStep( }; } + /** */ + long runDistributedUpdate( + String schemaName, + SqlFieldsQuery fieldsQry, + List cacheIds, + byte updateMode, + String tgtTable, + String[] colNames, + String selectQry, + GridQueryCancel cancel) { + + return rdcQryExec.update(schemaName, cacheIds, updateMode, tgtTable, colNames, selectQry, + fieldsQry.isEnforceJoinOrder(), fieldsQry.getPageSize(), fieldsQry.getTimeout(), + fieldsQry.getArgs(), fieldsQry.getPartitions(), cancel); + } + /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public QueryCursor> queryDistributedSql(String schemaName, SqlQuery qry, @@ -1365,33 +1381,13 @@ private Iterable> runQueryTwoStep( } } - LinkedHashSet caches0 = new LinkedHashSet<>(); - assert twoStepQry != null; - int tblCnt = twoStepQry.tablesCount(); + List cacheIds = collectCacheIds(mainCacheId, twoStepQry); - if (mainCacheId != null) - caches0.add(mainCacheId); - - if (tblCnt > 0) { - for (QueryTable tblKey : twoStepQry.tables()) { - GridH2Table tbl = dataTable(tblKey); - - int cacheId = CU.cacheId(tbl.cacheName()); - - caches0.add(cacheId); - } - } - - if (caches0.isEmpty()) + if (F.isEmpty(cacheIds)) twoStepQry.local(true); else { - //Prohibit usage indices with different numbers of segments in same query. - List cacheIds = new ArrayList<>(caches0); - - checkCacheIndexSegmentation(cacheIds); - twoStepQry.cacheIds(cacheIds); twoStepQry.local(qry.isLocal()); } @@ -1444,6 +1440,14 @@ private Iterable> runQueryTwoStep( return cursor; } + /** */ + public long mapDistributedUpdate(byte mode, String schemaName, String targetTable, String[] colNames, + String qry, Object[] params, int pageSize, int timeoutMillis, IndexingQueryFilter filter) + throws IgniteCheckedException { + return dmlProc.mapDistributedUpdate(mode, schemaName, targetTable, colNames, qry, params, pageSize, + timeoutMillis, filter); + } + /** * @throws IllegalStateException if segmented indices used with non-segmented indices. */ @@ -2392,6 +2396,37 @@ private int bindPartitionInfoParameter(CacheQueryPartitionInfo partInfo, Object[ U.close(conn, log); } + /** */ + public List collectCacheIds(@Nullable Integer mainCacheId, GridCacheTwoStepQuery twoStepQry) { + LinkedHashSet caches0 = new LinkedHashSet<>(); + + int tblCnt = twoStepQry.tablesCount(); + + if (mainCacheId != null) + caches0.add(mainCacheId); + + if (tblCnt > 0) { + for (QueryTable tblKey : twoStepQry.tables()) { + GridH2Table tbl = dataTable(tblKey); + + int cacheId = CU.cacheId(tbl.cacheName()); + + caches0.add(cacheId); + } + } + + if (caches0.isEmpty()) + return null; + else { + //Prohibit usage indices with different numbers of segments in same query. + List cacheIds = new ArrayList<>(caches0); + + checkCacheIndexSegmentation(cacheIds); + + return cacheIds; + } + } + /** * Closeable iterator. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index b81ac60025113..747756993e525 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.query.h2.dml; +import java.util.List; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.util.typedef.F; @@ -64,6 +65,12 @@ public final class UpdatePlan { /** Arguments for fast UPDATE or DELETE. */ public final FastUpdateArguments fastUpdateArgs; + /** Distributed update flag. */ + public boolean distributed = false; + + /** */ + public List cacheIds; + /** */ private UpdatePlan(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry, @@ -97,11 +104,12 @@ public static UpdatePlan forMerge(GridH2Table tbl, String[] colNames, int[] colT /** */ public static UpdatePlan forInsert(GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, - KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry, int rowsNum) { + KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry, + int rowsNum) { assert !F.isEmpty(colNames); - return new UpdatePlan(UpdateMode.INSERT, tbl, colNames, colTypes, keySupplier, valSupplier, keyColIdx, valColIdx, - selectQry, isLocSubqry, rowsNum, null); + return new UpdatePlan(UpdateMode.INSERT, tbl, colNames, colTypes, keySupplier, valSupplier, keyColIdx, + valColIdx, selectQry, isLocSubqry, rowsNum, null); } /** */ @@ -125,4 +133,10 @@ public static UpdatePlan forFastUpdate(UpdateMode mode, GridH2Table tbl, FastUpd return new UpdatePlan(mode, tbl, null, null, null, null, -1, -1, null, false, 0, fastUpdateArgs); } + /** */ + public static UpdatePlan fromMessage(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] colTypes, + String qry, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx) { + return new UpdatePlan(mode, tbl, colNames, colTypes, keySupplier, valSupplier, keyColIdx, valColIdx, qry, false, + 0, null); + } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java index b304109993ca9..2f9fcd6633615 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java @@ -212,7 +212,8 @@ else throw new IgniteSQLException("Unexpected DML operation [cls=" + stmt.getCla * @return Update plan. * @throws IgniteCheckedException if failed. */ - private static UpdatePlan planForUpdate(GridSqlStatement stmt, @Nullable Integer errKeysPos) throws IgniteCheckedException { + private static UpdatePlan planForUpdate(GridSqlStatement stmt, @Nullable Integer errKeysPos) + throws IgniteCheckedException { GridSqlElement target; FastUpdateArguments fastUpdate; @@ -299,6 +300,65 @@ else if (stmt instanceof GridSqlDelete) { } } + /** */ + public static UpdatePlan planFromMessage(GridCacheContext cctx, UpdateMode updateMode, GridH2Table tgtTbl, + GridH2RowDescriptor desc, String[] colNames, String qry) throws IgniteCheckedException { + boolean hasKeyProps = false; + boolean hasValProps = false; + int keyColIdx = -1; + int valColIdx = -1; + int[] colTypes = null; + + if (updateMode != UpdateMode.DELETE) { + colTypes = new int[colNames.length]; + + for (int i = 0; i < colTypes.length; ++i) { + String colName = colNames[i]; + + Column col = tgtTbl.getColumn(colName); + + colTypes[i] = col.getType(); + + int colId = col.getColumnId(); + + if (desc.isKeyColumn(colId)) { + keyColIdx = i; + + continue; + } + + if (desc.isValueColumn(colId)) { + valColIdx = i; + + continue; + } + + GridQueryProperty prop = desc.type().property(colName); + + assert prop != null : "Property '" + colName + "' not found."; + + if (prop.key()) + hasKeyProps = true; + else + hasValProps = true; + } + + if (updateMode == UpdateMode.UPDATE) + valColIdx = (valColIdx != -1) ? valColIdx + 2 : -1; + } + + KeyValueSupplier keySupplier = (updateMode == UpdateMode.INSERT || updateMode == UpdateMode.MERGE) ? + createSupplier(cctx, desc.type(), keyColIdx, hasKeyProps, true, false) : null; + + int valSupColIdx = (updateMode == UpdateMode.UPDATE)? ((valColIdx != -1) ? valColIdx : 1) : valColIdx; + + KeyValueSupplier valSupplier = (updateMode == UpdateMode.DELETE) ? null : + createSupplier(cctx, desc.type(), valSupColIdx, hasValProps, false, updateMode == UpdateMode.UPDATE); + + return UpdatePlan.fromMessage(updateMode, tgtTbl, colNames, colTypes, qry, keySupplier, valSupplier, + keyColIdx, valColIdx); + } + /** * Detect appropriate method of instantiating key or value (take from param, create binary builder, * invoke default ctor, or allocate). diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java new file mode 100644 index 0000000000000..ddd2d6ead3623 --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.twostep; + +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; +import javax.cache.CacheException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlResponse; + +/** */ +class DistributedUpdateRun { + /** */ + private CountDownLatch latch; + + /** */ + private AtomicLong updCntr = new AtomicLong(); + + /** */ + private volatile IgniteException ex; + + /** */ + DistributedUpdateRun(int nodeCount) { + latch = new CountDownLatch(nodeCount); + } + + /** + * @return Latch. + */ + CountDownLatch latch() { + return latch; + } + + /** */ + void handleNodeLeft(UUID nodeId) { + ex = new IgniteException("Update failed: node " + nodeId + " has left"); + + bringDownLatch(); + } + + /** */ + public boolean handleResponse(UUID id, GridH2DmlResponse msg) { + if (msg.status() == GridH2DmlResponse.STATUS_ERROR) { + ex = new IgniteException(msg.error()); + + bringDownLatch(); + + return true; + } + //TODO: handle errKeys + + this.updCntr.addAndGet(msg.updateCounter()); + + latch.countDown(); + + return latch.getCount() == 0L; + } + + private void bringDownLatch() { + while (latch.getCount() > 0) + latch.countDown(); + } + + /** */ + public long updateCounter() throws IgniteException { + + try { + latch.await(); + } + catch (InterruptedException e) { + throw new IgniteException(e); + } + //TODO: while timeout await/ while (!U.await(r.latch(), 500, TimeUnit.MILLISECONDS)) { + //TODO: throw error + //TODO: cancelation + if (ex != null) + throw ex; + + return updCntr.get(); + } + + /** */ + public void handleDisconnect(CacheException e) { + ex = new IgniteException(e); + + bringDownLatch(); + } +} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 0cc417281ddcd..d84b5c1cd2cb1 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -63,6 +63,8 @@ import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryFailResponse; import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryNextPageRequest; import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryNextPageResponse; +import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlRequest; +import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlResponse; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; import org.apache.ignite.internal.util.GridSpinBusyLock; import org.apache.ignite.internal.util.typedef.CI1; @@ -71,6 +73,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.thread.IgniteThread; +import org.apache.ignite.spi.indexing.IndexingQueryFilter; import org.h2.jdbc.JdbcResultSet; import org.h2.value.Value; import org.jetbrains.annotations.Nullable; @@ -208,6 +211,8 @@ else if (msg instanceof GridQueryNextPageRequest) onNextPageRequest(node, (GridQueryNextPageRequest)msg); else if (msg instanceof GridQueryCancelRequest) onCancel(node, (GridQueryCancelRequest)msg); + else if (msg instanceof GridH2DmlRequest) + onDmlRequest(node, (GridH2DmlRequest)msg); else processed = false; @@ -733,6 +738,65 @@ private void onQueryRequest0( } } + /** + * @param node Node. + * @param req DML request. + */ + private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) throws IgniteCheckedException { + int[] qryParts = req.queryPartitions(); + + final Map partsMap = req.partitions(); + + final int[] parts = qryParts == null ? partsMap == null ? null : partsMap.get(ctx.localNodeId()) : qryParts; + + final List cacheIds = req.caches(); + + AffinityTopologyVersion topVer = req.topologyVersion(); + + List reserved = new ArrayList<>(); + + if (!reservePartitions(cacheIds, topVer, parts, reserved)) { + log.error("Failed to reserve partitions for DML request"); + + GridH2DmlResponse rsp = new GridH2DmlResponse(req.requestId(), GridH2DmlResponse.STATUS_ERROR, -1, null, + "Failed to reserve partitions for DML request"); + + // Send response + ctx.io().sendToGridTopic(node, GridTopic.TOPIC_QUERY, rsp, QUERY_POOL); + } + + try { + String query = req.queries().get(0).query(); + + IndexingQueryFilter filter = h2.backupFilter(req.topologyVersion(), req.queryPartitions()); + + long updCntr = h2.mapDistributedUpdate(req.mode(), req.schemaName(), req.targetTable(), + req.columnNames(), query, req.parameters(), req.pageSize(), req.timeout(), filter); + + GridH2DmlResponse rsp = new GridH2DmlResponse(req.requestId(), GridH2DmlResponse.STATUS_OK, updCntr, null, + null); + + // Send response + ctx.io().sendToGridTopic(node, GridTopic.TOPIC_QUERY, rsp, QUERY_POOL); + } + catch (Exception e) { + log.error("Error processing dml request " + e.toString()); + + GridH2DmlResponse rsp = new GridH2DmlResponse(req.requestId(), GridH2DmlResponse.STATUS_ERROR, -1, null, + e.getMessage()); + + // Send response + ctx.io().sendToGridTopic(node, GridTopic.TOPIC_QUERY, rsp, QUERY_POOL); + } + finally { + if (!F.isEmpty(reserved)) { + // Release reserved partitions. + for (int i = 0; i < reserved.size(); i++) + reserved.get(i).release(); + } + } + } + /** * @param node Node. * @param qryReqId Query request ID. diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index 8638794f8785e..9b61956605c37 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -74,6 +74,8 @@ import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryFailResponse; import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryNextPageRequest; import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryNextPageResponse; +import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlRequest; +import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlResponse; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest; import org.apache.ignite.internal.util.GridIntIterator; import org.apache.ignite.internal.util.GridIntList; @@ -83,6 +85,7 @@ import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiClosure; +import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.plugin.extensions.communication.Message; import org.h2.command.ddl.CreateTableData; @@ -130,6 +133,9 @@ public class GridReduceQueryExecutor { /** */ private final ConcurrentMap runs = new ConcurrentHashMap8<>(); + /** */ + private final ConcurrentMap updRuns = new ConcurrentHashMap8<>(); + /** */ private volatile List fakeTbls = Collections.emptyList(); @@ -197,6 +203,10 @@ public void start(final GridKernalContext ctx, final IgniteH2Indexing h2) throws } } } + + for (DistributedUpdateRun r : updRuns.values()) + r.handleNodeLeft(nodeId); + } }, EventType.EVT_NODE_FAILED, EventType.EVT_NODE_LEFT); } @@ -229,6 +239,8 @@ public void onMessage(UUID nodeId, Object msg) { onNextPage(node, (GridQueryNextPageResponse)msg); else if (msg instanceof GridQueryFailResponse) onFail(node, (GridQueryFailResponse)msg); + else if (msg instanceof GridH2DmlResponse) + onDmlResponse(node, (GridH2DmlResponse)msg); else processed = false; @@ -844,6 +856,88 @@ public Iterator> query( } } + /** */ + public long update( + String schemaName, + List cacheIds, + byte updateMode, + String tgtTable, + String[] colNames, + String selectQry, + boolean enforceJoinOrder, + int pageSize, + int timeoutMillis, + Object[] params, + final int[] parts, + GridQueryCancel cancel + ) { + AffinityTopologyVersion topVer = h2.readyTopologyVersion(); + + IgniteBiTuple, Map> nap = + nodesForPartitions(cacheIds, topVer, parts, false); + + Collection nodes = nap.get1(); + Map partsMap = nap.get2(); + + DistributedUpdateRun r = new DistributedUpdateRun(nodes.size()); + + final long reqId = qryIdGen.incrementAndGet(); + + GridH2DmlRequest req = new GridH2DmlRequest() + .requestId(reqId) + .topologyVersion(h2.readyTopologyVersion()) + .mode(updateMode) + .schemaName(schemaName) + .targetTable(tgtTable) + .columnNames(colNames) + .queries(Collections.singletonList(new GridCacheSqlQuery(selectQry))) + .pageSize(pageSize) + .parameters(params) + .timeout(timeoutMillis) + .flags(enforceJoinOrder ? GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER : 0); + + updRuns.put(reqId, r); + + try { + if (send(nodes, req, parts == null ? null : new ExplicitPartitionsSpecializer(partsMap), false)) + awaitAllReplies(r, nodes, cancel); + + return r.updateCounter(); + } + catch (IgniteCheckedException | RuntimeException e) { + log.error("Error in update: " + e.toString()); + + throw new CacheException("Failed to run update: ", e); + } + finally { + //TODO: cancel remote updates + + if (!updRuns.remove(reqId, r)) + U.warn(log, "Update run was already removed: " + reqId); + } + } + + /** */ + private void onDmlResponse(final ClusterNode node, GridH2DmlResponse msg) { + try { + long reqId = msg.requestId(); + + DistributedUpdateRun r = updRuns.get(reqId); + + if (r == null) { + log.error("Unknown dml request with id " + reqId); + + return; + } + + if (r.handleResponse(node.id(), msg)) + updRuns.remove(reqId); + } + catch (Exception e) { + log.error("Error in dml response processing: " + e.toString()); + } + } + /** * @param cacheIds Cache IDs. * @return The first partitioned cache context. @@ -923,6 +1017,25 @@ private void awaitAllReplies(ReduceQueryRun r, Collection nodes, Gr } } + /** */ + private void awaitAllReplies(DistributedUpdateRun r, Collection nodes, GridQueryCancel cancel) + throws IgniteInterruptedCheckedException, QueryCancelledException { + while (!U.await(r.latch(), 500, TimeUnit.MILLISECONDS)) { + + cancel.checkCancelled(); + + for (ClusterNode node : nodes) { + if (!ctx.discovery().alive(node)) { + r.handleNodeLeft(node.id()); + + assert r.latch().getCount() == 0; + + return; + } + } + } + } + /** * Gets or creates new fake table for index. * @@ -1308,6 +1421,37 @@ private static Map convert(Map m) { return res; } + // TODO: reuse this function in query() + /** */ + private IgniteBiTuple, Map> nodesForPartitions( + List cacheIds, AffinityTopologyVersion topVer, int[] parts, boolean isReplicatedOnly) { + Collection nodes = null; + Map partsMap = null; + Map qryMap = null; + + if (isPreloadingActive(cacheIds)) { + if (isReplicatedOnly) + nodes = replicatedUnstableDataNodes(cacheIds); + else { + partsMap = partitionedUnstableDataNodes(cacheIds); + + if (partsMap != null) { + qryMap = narrowForQuery(partsMap, parts); + + nodes = qryMap == null ? null : qryMap.keySet(); + } + } + } + else { + qryMap = stableDataNodes(isReplicatedOnly, topVer, cacheIds, parts); + + if (qryMap != null) + nodes = qryMap.keySet(); + } + + return new IgniteBiTuple<>(nodes, qryMap == null ? partsMap : qryMap); + } + /** * @param conn Connection. * @param qry Query. @@ -1403,6 +1547,9 @@ public void onDisconnected(IgniteFuture reconnectFut) { for (Map.Entry e : runs.entrySet()) e.getValue().disconnected(err); + + for (DistributedUpdateRun r: updRuns.values()) + r.handleDisconnect(err); } /** @@ -1436,6 +1583,8 @@ public void cancelQueries(Collection queries) { if (run != null) run.queryInfo().cancel(); } + + // TODO: do we need updRuns processed here (and in longRunningQueries) as well? } /** */ @@ -1478,11 +1627,21 @@ public ExplicitPartitionsSpecializer(Map partsMap) { /** {@inheritDoc} */ @Override public Message apply(ClusterNode node, Message msg) { - GridH2QueryRequest rq = new GridH2QueryRequest((GridH2QueryRequest)msg); + if (msg instanceof GridH2QueryRequest) { + GridH2QueryRequest rq = new GridH2QueryRequest((GridH2QueryRequest)msg); - rq.queryPartitions(toArray(partsMap.get(node))); + rq.queryPartitions(toArray(partsMap.get(node))); + + return rq; + } else if (msg instanceof GridH2DmlRequest) { + GridH2DmlRequest rq = new GridH2DmlRequest((GridH2DmlRequest)msg); + + rq.queryPartitions(toArray(partsMap.get(node))); + + return rq; + } - return rq; + return msg; } } } \ No newline at end of file diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java new file mode 100644 index 0000000000000..e878767576b71 --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java @@ -0,0 +1,697 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.processors.query.h2.twostep.msg; + +import java.io.Externalizable; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.GridDirectCollection; +import org.apache.ignite.internal.GridDirectMap; +import org.apache.ignite.internal.GridDirectTransient; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.binary.BinaryMarshaller; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable; +import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; +import org.apache.ignite.internal.processors.cache.query.QueryTable; +import org.apache.ignite.internal.util.tostring.GridToStringInclude; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.marshaller.Marshaller; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType; +import org.apache.ignite.plugin.extensions.communication.MessageReader; +import org.apache.ignite.plugin.extensions.communication.MessageWriter; + +import static org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery.EMPTY_PARAMS; + +/** */ +public class GridH2DmlRequest implements Message, GridCacheQueryMarshallable { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private long reqId; + + /** */ + @GridToStringInclude + @GridDirectCollection(Integer.class) + private List caches; + + /** Topology version. */ + private AffinityTopologyVersion topVer; + + /** Explicit partitions mappings for nodes. */ + @GridToStringInclude + @GridDirectMap(keyType = UUID.class, valueType = int[].class) + private Map parts; + + /** Query partitions. */ + @GridToStringInclude + private int[] qryParts; + + /** */ + private int pageSize; + + /** */ + @GridToStringInclude + @GridDirectCollection(Message.class) + private List qrys; + + /** */ + private byte flags; + + /** */ + @GridToStringInclude + @GridDirectCollection(Message.class) + private Collection tbls; + + /** */ + private int timeout; + + /** */ + @GridToStringInclude(sensitive = true) + @GridDirectTransient + private Object[] params; + + /** */ + private byte[] paramsBytes; + + /** Schema name. */ + private String schemaName; + + /** Update mode. */ + private byte mode; + + /** Target table. */ + private String tgtTbl; //TODO: go for QueryTable instead? + + /** Target table column names. */ + private String[] colNames; + + /** + * Required by {@link Externalizable} + */ + public GridH2DmlRequest() { + // No-op. + } + + /** + * @param req Request. + */ + public GridH2DmlRequest(GridH2DmlRequest req) { + reqId = req.reqId; + caches = req.caches; + topVer = req.topVer; + parts = req.parts; + qryParts = req.qryParts; + pageSize = req.pageSize; + qrys = req.qrys; + flags = req.flags; + tbls = req.tbls; + timeout = req.timeout; + params = req.params; + paramsBytes = req.paramsBytes; + schemaName = req.schemaName; + mode = req.mode; + tgtTbl = req.tgtTbl; + colNames = req.colNames; + } + + /** + * @return Parameters. + */ + public Object[] parameters() { + return params; + } + + /** + * @param params Parameters. + * @return {@code this}. + */ + public GridH2DmlRequest parameters(Object[] params) { + if (params == null) + params = EMPTY_PARAMS; + + this.params = params; + + return this; + } + + /** + * @param tbls Tables. + * @return {@code this}. + */ + public GridH2DmlRequest tables(Collection tbls) { + this.tbls = tbls; + + return this; + } + + /** + * @return Tables. + */ + public Collection tables() { + return tbls; + } + + /** + * @param reqId Request ID. + * @return {@code this}. + */ + public GridH2DmlRequest requestId(long reqId) { + this.reqId = reqId; + + return this; + } + + /** + * @return Request ID. + */ + public long requestId() { + return reqId; + } + + /** + * @param caches Caches. + * @return {@code this}. + */ + public GridH2DmlRequest caches(List caches) { + this.caches = caches; + + return this; + } + + /** + * @return Caches. + */ + public List caches() { + return caches; + } + + /** + * @param topVer Topology version. + * @return {@code this}. + */ + public GridH2DmlRequest topologyVersion(AffinityTopologyVersion topVer) { + this.topVer = topVer; + + return this; + } + + /** + * @return Topology version. + */ + public AffinityTopologyVersion topologyVersion() { + return topVer; + } + + /** + * @return Explicit partitions mapping. + */ + public Map partitions() { + return parts; + } + + /** + * @param parts Explicit partitions mapping. + * @return {@code this}. + */ + public GridH2DmlRequest partitions(Map parts) { + this.parts = parts; + + return this; + } + + /** + * @return Query partitions. + */ + public int[] queryPartitions() { + return qryParts; + } + + /** + * @param qryParts Query partitions. + * @return {@code this}. + */ + public GridH2DmlRequest queryPartitions(int[] qryParts) { + this.qryParts = qryParts; + + return this; + } + + /** + * @param pageSize Page size. + * @return {@code this}. + */ + public GridH2DmlRequest pageSize(int pageSize) { + this.pageSize = pageSize; + + return this; + } + + /** + * @return Page size. + */ + public int pageSize() { + return pageSize; + } + + /** + * @param qrys SQL Queries. + * @return {@code this}. + */ + public GridH2DmlRequest queries(List qrys) { + this.qrys = qrys; + + return this; + } + + /** + * @return SQL Queries. + */ + public List queries() { + return qrys; + } + + /** + * @param flags Flags. + * @return {@code this}. + */ + public GridH2DmlRequest flags(int flags) { + assert flags >= 0 && flags <= 255: flags; + + this.flags = (byte)flags; + + return this; + } + + /** + * @param flags Flags to check. + * @return {@code true} If all the requested flags are set to {@code true}. + */ + public boolean isFlagSet(int flags) { + return (this.flags & flags) == flags; + } + + /** + * @return Timeout. + */ + public int timeout() { + return timeout; + } + + /** + * @param timeout New timeout. + * @return {@code this}. + */ + public GridH2DmlRequest timeout(int timeout) { + this.timeout = timeout; + + return this; + } + + /** + * @return Schema name. + */ + public String schemaName() { + return schemaName; + } + + /** + * @param schemaName Schema name. + * @return {@code this}. + */ + public GridH2DmlRequest schemaName(String schemaName) { + this.schemaName = schemaName; + + return this; + } + + /** + * @return Update mode. + */ + public byte mode() { + return mode; + } + + /** + * @param mode Update mode. + * @return {@code this} + */ + public GridH2DmlRequest mode(byte mode) { + this.mode = mode; + + return this; + } + + /** + * @return Target table. + */ + public String targetTable() { + return tgtTbl; + } + + /** + * @param targetTable Target table. + * @return {@code this} + */ + public GridH2DmlRequest targetTable(String targetTable) { + tgtTbl = targetTable; + + return this; + } + + /** + * @return Target table column names. + */ + public String[] columnNames() { + return colNames; + } + + /** + * @param columnNames Target table column names. + * @return {@code this} + */ + public GridH2DmlRequest columnNames(String[] columnNames) { + this.colNames = columnNames; + + return this; + } + + /** {@inheritDoc} */ + @Override public void marshall(Marshaller m) { + if (paramsBytes != null) + return; + + assert params != null; + + try { + paramsBytes = U.marshal(m, params); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + + /** {@inheritDoc} */ + @SuppressWarnings("IfMayBeConditional") + @Override public void unmarshall(Marshaller m, GridKernalContext ctx) { + if (params != null) + return; + + assert paramsBytes != null; + + try { + final ClassLoader ldr = U.resolveClassLoader(ctx.config()); + + if (m instanceof BinaryMarshaller) + // To avoid deserializing of enum types. + params = ((BinaryMarshaller)m).binaryMarshaller().unmarshal(paramsBytes, ldr); + else + params = U.unmarshal(m, paramsBytes, ldr); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + + /** {@inheritDoc} */ + @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) { + writer.setBuffer(buf); + + if (!writer.isHeaderWritten()) { + if (!writer.writeHeader(directType(), fieldsCount())) + return false; + + writer.onHeaderWritten(); + } + + switch (writer.state()) { + case 0: + if (!writer.writeCollection("caches", caches, MessageCollectionItemType.INT)) + return false; + + writer.incrementState(); + + case 1: + if (!writer.writeObjectArray("colNames", colNames, MessageCollectionItemType.STRING)) + return false; + + writer.incrementState(); + + case 2: + if (!writer.writeByte("flags", flags)) + return false; + + writer.incrementState(); + + case 3: + if (!writer.writeByte("mode", mode)) + return false; + + writer.incrementState(); + + case 4: + if (!writer.writeInt("pageSize", pageSize)) + return false; + + writer.incrementState(); + + case 5: + if (!writer.writeByteArray("paramsBytes", paramsBytes)) + return false; + + writer.incrementState(); + + case 6: + if (!writer.writeMap("parts", parts, MessageCollectionItemType.UUID, MessageCollectionItemType.INT_ARR)) + return false; + + writer.incrementState(); + + case 7: + if (!writer.writeIntArray("qryParts", qryParts)) + return false; + + writer.incrementState(); + + case 8: + if (!writer.writeCollection("qrys", qrys, MessageCollectionItemType.MSG)) + return false; + + writer.incrementState(); + + case 9: + if (!writer.writeLong("reqId", reqId)) + return false; + + writer.incrementState(); + + case 10: + if (!writer.writeString("schemaName", schemaName)) + return false; + + writer.incrementState(); + + case 11: + if (!writer.writeCollection("tbls", tbls, MessageCollectionItemType.MSG)) + return false; + + writer.incrementState(); + + case 12: + if (!writer.writeString("tgtTbl", tgtTbl)) + return false; + + writer.incrementState(); + + case 13: + if (!writer.writeInt("timeout", timeout)) + return false; + + writer.incrementState(); + + case 14: + if (!writer.writeMessage("topVer", topVer)) + return false; + + writer.incrementState(); + + } + + return true; + } + + /** {@inheritDoc} */ + @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) { + reader.setBuffer(buf); + + if (!reader.beforeMessageRead()) + return false; + + switch (reader.state()) { + case 0: + caches = reader.readCollection("caches", MessageCollectionItemType.INT); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 1: + colNames = reader.readObjectArray("colNames", MessageCollectionItemType.STRING, String.class); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 2: + flags = reader.readByte("flags"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 3: + mode = reader.readByte("mode"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 4: + pageSize = reader.readInt("pageSize"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 5: + paramsBytes = reader.readByteArray("paramsBytes"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 6: + parts = reader.readMap("parts", MessageCollectionItemType.UUID, MessageCollectionItemType.INT_ARR, false); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 7: + qryParts = reader.readIntArray("qryParts"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 8: + qrys = reader.readCollection("qrys", MessageCollectionItemType.MSG); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 9: + reqId = reader.readLong("reqId"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 10: + schemaName = reader.readString("schemaName"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 11: + tbls = reader.readCollection("tbls", MessageCollectionItemType.MSG); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 12: + tgtTbl = reader.readString("tgtTbl"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 13: + timeout = reader.readInt("timeout"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 14: + topVer = reader.readMessage("topVer"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + } + + return reader.afterMessageRead(GridH2DmlRequest.class); + } + + /** {@inheritDoc} */ + @Override public short directType() { + return -55; + } + + /** {@inheritDoc} */ + @Override public byte fieldsCount() { + return 15; + } + + /** {@inheritDoc} */ + @Override public void onAckReceived() { + // No-op. + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(GridH2DmlRequest.class, this); + } +} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java new file mode 100644 index 0000000000000..02e98db46e709 --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2.twostep.msg; + +import java.nio.ByteBuffer; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType; +import org.apache.ignite.plugin.extensions.communication.MessageReader; +import org.apache.ignite.plugin.extensions.communication.MessageWriter; + +/** */ +public class GridH2DmlResponse implements Message { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + public static final byte STATUS_OK = 0; + + /** */ + public static final byte STATUS_ERROR = 1; + + /** */ + public static final byte STATUS_ERR_KEYS = 2; + + /** */ + private long reqId; + + /** */ + private byte status; + + /** */ + private long updCnt; + + /** */ + private String err; + + /** */ + private Object[] errKeys; + + /** + * Default constructor. + */ + public GridH2DmlResponse() { + // No-op. + } + + /** */ + public GridH2DmlResponse(long reqId, byte status, long updCnt, Object[] errKeys, String error) { + this.reqId = reqId; + this.status = status; + this.updCnt = updCnt; + this.errKeys = errKeys; + this.err = error; + } + + /** + * @return Request id. + */ + public long requestId() { + return reqId; + } + + /** + * @return Status. + */ + public byte status() { + return status; + } + + /** + * @return Update counter. + */ + public long updateCounter() { + return updCnt; + } + + /** + * @return Error keys. + */ + public Object[] errorKeys() { + return errKeys; + } + + /** + * @return Error message. + */ + public String error() { + return err; + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(GridH2DmlResponse.class, this); + } + + /** {@inheritDoc} */ + @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) { + writer.setBuffer(buf); + + if (!writer.isHeaderWritten()) { + if (!writer.writeHeader(directType(), fieldsCount())) + return false; + + writer.onHeaderWritten(); + } + + switch (writer.state()) { + case 0: + if (!writer.writeString("err", err)) + return false; + + writer.incrementState(); + + case 1: + if (!writer.writeObjectArray("errKeys", errKeys, MessageCollectionItemType.MSG)) + return false; + + writer.incrementState(); + + case 2: + if (!writer.writeLong("reqId", reqId)) + return false; + + writer.incrementState(); + + case 3: + if (!writer.writeByte("status", status)) + return false; + + writer.incrementState(); + + case 4: + if (!writer.writeLong("updCnt", updCnt)) + return false; + + writer.incrementState(); + + } + + return true; + } + + /** {@inheritDoc} */ + @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) { + reader.setBuffer(buf); + + if (!reader.beforeMessageRead()) + return false; + + switch (reader.state()) { + case 0: + err = reader.readString("err"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 1: + errKeys = reader.readObjectArray("errKeys", MessageCollectionItemType.MSG, Object.class); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 2: + reqId = reader.readLong("reqId"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 3: + status = reader.readByte("status"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + case 4: + updCnt = reader.readLong("updCnt"); + + if (!reader.isLastRead()) + return false; + + reader.incrementState(); + + } + + return reader.afterMessageRead(GridH2DmlResponse.class); + } + + /** {@inheritDoc} */ + @Override public short directType() { + return -56; + } + + /** {@inheritDoc} */ + @Override public byte fieldsCount() { + return 5; + } + + @Override public void onAckReceived() { + // No-op + } +} + diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2ValueMessageFactory.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2ValueMessageFactory.java index 18b1afbaa3857..3c133928ef7c7 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2ValueMessageFactory.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2ValueMessageFactory.java @@ -112,6 +112,12 @@ public class GridH2ValueMessageFactory implements MessageFactory { case -54: return new QueryTable(); + + case -55: + return new GridH2DmlRequest(); + + case -56: + return new GridH2DmlResponse(); } return null; From e88e20220dee7d4e03cbdd50925e35f19953400e Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Mon, 21 Aug 2017 20:11:40 +0300 Subject: [PATCH 02/40] IGNITE-6024: fixing todos --- .../query/h2/DmlStatementsProcessor.java | 32 +--- .../processors/query/h2/IgniteH2Indexing.java | 4 +- .../processors/query/h2/UpdateResult.java | 58 +++++++ .../h2/twostep/DistributedUpdateRun.java | 109 ++++++------ .../h2/twostep/GridMapQueryExecutor.java | 45 +++-- .../h2/twostep/GridReduceQueryExecutor.java | 162 +++++++++++------- 6 files changed, 256 insertions(+), 154 deletions(-) create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index b7b90ddd007fb..254948f2587e3 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -215,6 +215,7 @@ else if (items == 0L) return new UpdateResult(items, errKeys); } + /** */ private boolean checkDistributed(String schemaName, UpdatePlan p, SqlFieldsQuery fieldsQry) { Connection c = idx.connectionForSchema(schemaName); @@ -571,10 +572,8 @@ private UpdateResult doDistributedUpdate(String schemaName, SqlFieldsQuery field if (cancel == null) cancel = new GridQueryCancel(); - long cnt = idx.runDistributedUpdate(schemaName, fieldsQry, plan.cacheIds, (byte)plan.mode.ordinal(), + return idx.runDistributedUpdate(schemaName, fieldsQry, plan.cacheIds, (byte)plan.mode.ordinal(), plan.tbl.getName(), plan.colNames, plan.selectQry, cancel); - - return new UpdateResult(cnt, null); } /** @@ -1048,7 +1047,7 @@ private static PageProcessingResult processPage(GridCacheContext cctx, } /** */ - long mapDistributedUpdate(byte mode, String schemaName, String targetTable, + UpdateResult mapDistributedUpdate(byte mode, String schemaName, String targetTable, String[] colNames, String qry, Object[] params, int pageSize, int timeoutMillis, IndexingQueryFilter filter) throws IgniteCheckedException { GridH2Table tgtTbl = idx.dataTable(schemaName, targetTable); //TODO: use QueryTable instead @@ -1093,7 +1092,7 @@ else if (!opCtx.isKeepBinary()) cctx.operationContextPerCall(opCtx); } - return res.cnt; + return res; } /** */ @@ -1192,29 +1191,6 @@ static boolean isDmlStatement(Prepared stmt) { return stmt instanceof Merge || stmt instanceof Insert || stmt instanceof Update || stmt instanceof Delete; } - /** Update result - modifications count and keys to re-run query with, if needed. */ - private final static class UpdateResult { - /** Result to return for operations that affected 1 item - mostly to be used for fast updates and deletes. */ - final static UpdateResult ONE = new UpdateResult(1, X.EMPTY_OBJECT_ARRAY); - - /** Result to return for operations that affected 0 items - mostly to be used for fast updates and deletes. */ - final static UpdateResult ZERO = new UpdateResult(0, X.EMPTY_OBJECT_ARRAY); - - /** Number of processed items. */ - final long cnt; - - /** Keys that failed to be updated or deleted due to concurrent modification of values. */ - @NotNull - final Object[] errKeys; - - /** */ - @SuppressWarnings("ConstantConditions") - private UpdateResult(long cnt, Object[] errKeys) { - this.cnt = cnt; - this.errKeys = U.firstNotNull(errKeys, X.EMPTY_OBJECT_ARRAY); - } - } - /** Result of processing an individual page with {@link IgniteCache#invokeAll} including error details, if any. */ private final static class PageProcessingResult { /** Number of successfully processed items. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 39b72398d2268..1d3858af777f7 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1193,7 +1193,7 @@ private Iterable> runQueryTwoStep( } /** */ - long runDistributedUpdate( + UpdateResult runDistributedUpdate( String schemaName, SqlFieldsQuery fieldsQry, List cacheIds, @@ -1441,7 +1441,7 @@ long runDistributedUpdate( } /** */ - public long mapDistributedUpdate(byte mode, String schemaName, String targetTable, String[] colNames, + public UpdateResult mapDistributedUpdate(byte mode, String schemaName, String targetTable, String[] colNames, String qry, Object[] params, int pageSize, int timeoutMillis, IndexingQueryFilter filter) throws IgniteCheckedException { return dmlProc.mapDistributedUpdate(mode, schemaName, targetTable, colNames, qry, params, pageSize, diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java new file mode 100644 index 0000000000000..1ab368d15e8da --- /dev/null +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.h2; + +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.jetbrains.annotations.NotNull; + +/** Update result - modifications count and keys to re-run query with, if needed. */ +public final class UpdateResult { + /** Result to return for operations that affected 1 item - mostly to be used for fast updates and deletes. */ + final static UpdateResult ONE = new UpdateResult(1, X.EMPTY_OBJECT_ARRAY); + + /** Result to return for operations that affected 0 items - mostly to be used for fast updates and deletes. */ + final static UpdateResult ZERO = new UpdateResult(0, X.EMPTY_OBJECT_ARRAY); + + /** Number of processed items. */ + final long cnt; + + /** Keys that failed to be updated or deleted due to concurrent modification of values. */ + @NotNull + final Object[] errKeys; + + /** */ + public @SuppressWarnings("ConstantConditions") UpdateResult(long cnt, Object[] errKeys) { + this.cnt = cnt; + this.errKeys = U.firstNotNull(errKeys, X.EMPTY_OBJECT_ARRAY); + } + + /** + * @return Update counter. + */ + public long counter() { + return cnt; + } + + /** + * @return Error keys. + */ + public Object[] errorKeys() { + return errKeys; + } +} diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java index ddd2d6ead3623..f48f88d11640f 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java @@ -17,88 +17,97 @@ package org.apache.ignite.internal.processors.query.h2.twostep; +import java.util.Arrays; import java.util.UUID; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicLong; import javax.cache.CacheException; -import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.processors.query.GridRunningQueryInfo; +import org.apache.ignite.internal.processors.query.h2.UpdateResult; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlResponse; +import org.apache.ignite.internal.util.GridConcurrentHashSet; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.F; /** */ class DistributedUpdateRun { - /** */ - private CountDownLatch latch; + /** Response counter. */ + private AtomicLong rspCntr; - /** */ + /** Update counter. */ private AtomicLong updCntr = new AtomicLong(); - /** */ - private volatile IgniteException ex; + /** Error keys. */ + private GridConcurrentHashSet errorKeys = new GridConcurrentHashSet<>(); + + /** Query info. */ + private final GridRunningQueryInfo qry; + + /** Result future. */ + private GridFutureAdapter fut = new GridFutureAdapter<>(); /** */ - DistributedUpdateRun(int nodeCount) { - latch = new CountDownLatch(nodeCount); + DistributedUpdateRun(int nodeCount, GridRunningQueryInfo qry) { + rspCntr = new AtomicLong(nodeCount); + + this.qry = qry; } /** - * @return Latch. + * @return Query info. */ - CountDownLatch latch() { - return latch; + GridRunningQueryInfo queryInfo() { + return qry; } - /** */ - void handleNodeLeft(UUID nodeId) { - ex = new IgniteException("Update failed: node " + nodeId + " has left"); + /** + * @return Result future. + */ + GridFutureAdapter future() { + return fut; + } - bringDownLatch(); + /** */ + void handleDisconnect(CacheException e) { + fut.onDone(new IgniteCheckedException("Update failed.", e)); } /** */ - public boolean handleResponse(UUID id, GridH2DmlResponse msg) { - if (msg.status() == GridH2DmlResponse.STATUS_ERROR) { - ex = new IgniteException(msg.error()); + void handleNodeLeft(UUID nodeId) { + fut.onDone(new IgniteCheckedException("Update failed: node " + nodeId + " has left")); + } - bringDownLatch(); + /** */ + boolean handleResponse(UUID id, GridH2DmlResponse msg) { + boolean done = rspCntr.decrementAndGet() == 0; - return true; - } - //TODO: handle errKeys + switch (msg.status()) { + case GridH2DmlResponse.STATUS_ERR_KEYS: + updCntr.addAndGet(msg.updateCounter()); - this.updCntr.addAndGet(msg.updateCounter()); + if (!F.isEmpty(msg.errorKeys())) + errorKeys.addAll(Arrays.asList(msg.errorKeys())); - latch.countDown(); + // Intentional fall-through. + case GridH2DmlResponse.STATUS_OK: + updCntr.addAndGet(msg.updateCounter()); - return latch.getCount() == 0L; - } + if (done) + fut.onDone(new UpdateResult(updCntr.get(), errorKeys.toArray())); - private void bringDownLatch() { - while (latch.getCount() > 0) - latch.countDown(); - } + return done; - /** */ - public long updateCounter() throws IgniteException { + case GridH2DmlResponse.STATUS_ERROR: + String err = msg.error(); - try { - latch.await(); - } - catch (InterruptedException e) { - throw new IgniteException(e); - } - //TODO: while timeout await/ while (!U.await(r.latch(), 500, TimeUnit.MILLISECONDS)) { - //TODO: throw error - //TODO: cancelation - if (ex != null) - throw ex; + fut.onDone(new IgniteCheckedException("Update failed. " + (F.isEmpty(err)? "" : err))); - return updCntr.get(); - } + return false; - /** */ - public void handleDisconnect(CacheException e) { - ex = new IgniteException(e); + default: + fut.onDone(new IgniteCheckedException("Update failed. Invalid status in response message.")); - bringDownLatch(); + return false; + } } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index d84b5c1cd2cb1..8019536798603 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -56,6 +56,7 @@ import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; import org.apache.ignite.internal.processors.query.h2.H2Utils; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; +import org.apache.ignite.internal.processors.query.h2.UpdateResult; import org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode; import org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryContext; import org.apache.ignite.internal.processors.query.h2.opt.GridH2RetryException; @@ -758,11 +759,9 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th if (!reservePartitions(cacheIds, topVer, parts, reserved)) { log.error("Failed to reserve partitions for DML request"); - GridH2DmlResponse rsp = new GridH2DmlResponse(req.requestId(), GridH2DmlResponse.STATUS_ERROR, -1, null, - "Failed to reserve partitions for DML request"); + sendUpdateResponse(node, req.requestId(), null, "Failed to reserve partitions for DML request"); - // Send response - ctx.io().sendToGridTopic(node, GridTopic.TOPIC_QUERY, rsp, QUERY_POOL); + return; } try { @@ -770,23 +769,16 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th IndexingQueryFilter filter = h2.backupFilter(req.topologyVersion(), req.queryPartitions()); - long updCntr = h2.mapDistributedUpdate(req.mode(), req.schemaName(), req.targetTable(), + UpdateResult updRes = h2.mapDistributedUpdate(req.mode(), req.schemaName(), req.targetTable(), req.columnNames(), query, req.parameters(), req.pageSize(), req.timeout(), filter); - GridH2DmlResponse rsp = new GridH2DmlResponse(req.requestId(), GridH2DmlResponse.STATUS_OK, updCntr, null, - null); + sendUpdateResponse(node, req.requestId(), updRes, null); - // Send response - ctx.io().sendToGridTopic(node, GridTopic.TOPIC_QUERY, rsp, QUERY_POOL); } catch (Exception e) { log.error("Error processing dml request " + e.toString()); - GridH2DmlResponse rsp = new GridH2DmlResponse(req.requestId(), GridH2DmlResponse.STATUS_ERROR, -1, null, - e.getMessage()); - - // Send response - ctx.io().sendToGridTopic(node, GridTopic.TOPIC_QUERY, rsp, QUERY_POOL); + sendUpdateResponse(node, req.requestId(), null, e.getMessage()); } finally { if (!F.isEmpty(reserved)) { @@ -821,6 +813,31 @@ private void sendError(ClusterNode node, long qryReqId, Throwable err) { } } + /** + * + * @param node + * @param reqId + * @param updResult + * @param error + */ + private void sendUpdateResponse(ClusterNode node, long reqId, UpdateResult updResult, String error) { + try { + byte status = (error != null) ? GridH2DmlResponse.STATUS_ERROR : + F.isEmpty(updResult.errorKeys()) ? GridH2DmlResponse.STATUS_OK : GridH2DmlResponse.STATUS_ERR_KEYS; + + GridH2DmlResponse rsp = new GridH2DmlResponse(reqId, status, updResult.counter(), updResult.errorKeys(), + error); + + if (node.isLocal()) + h2.reduceQueryExecutor().onMessage(ctx.localNodeId(), rsp); + else + ctx.io().sendToGridTopic(node, GridTopic.TOPIC_QUERY, rsp, QUERY_POOL); + } + catch (Exception e) { + U.error(log, "Failed to send message.", e); + } + } + /** * @param node Node. * @param req Request. diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index 9b61956605c37..680a373176338 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -59,6 +59,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable; +import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType; import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery; import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator; @@ -67,6 +68,7 @@ import org.apache.ignite.internal.processors.query.h2.H2FieldsIterator; import org.apache.ignite.internal.processors.query.h2.H2Utils; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; +import org.apache.ignite.internal.processors.query.h2.UpdateResult; import org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryContext; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlSortColumn; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlType; @@ -85,7 +87,6 @@ import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiClosure; -import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.plugin.extensions.communication.Message; import org.h2.command.ddl.CreateTableData; @@ -587,25 +588,11 @@ public Iterator> query( if (qry.isLocal()) nodes = singletonList(ctx.discovery().localNode()); else { - if (isPreloadingActive(cacheIds)) { - if (isReplicatedOnly) - nodes = replicatedUnstableDataNodes(cacheIds); - else { - partsMap = partitionedUnstableDataNodes(cacheIds); - - if (partsMap != null) { - qryMap = narrowForQuery(partsMap, parts); + NodesForPartitionsResult nodesParts = nodesForPartitions(cacheIds, topVer, parts, isReplicatedOnly); - nodes = qryMap == null ? null : qryMap.keySet(); - } - } - } - else { - qryMap = stableDataNodes(isReplicatedOnly, topVer, cacheIds, parts); - - if (qryMap != null) - nodes = qryMap.keySet(); - } + nodes = nodesParts.nodes(); + partsMap = nodesParts.partitionsMap(); + qryMap = nodesParts.queryPartitionsMap(); if (nodes == null) continue; // Retry. @@ -857,7 +844,7 @@ public Iterator> query( } /** */ - public long update( + public UpdateResult update( String schemaName, List cacheIds, byte updateMode, @@ -873,15 +860,16 @@ public long update( ) { AffinityTopologyVersion topVer = h2.readyTopologyVersion(); - IgniteBiTuple, Map> nap = - nodesForPartitions(cacheIds, topVer, parts, false); + NodesForPartitionsResult nodesParts = nodesForPartitions(cacheIds, topVer, parts, false); - Collection nodes = nap.get1(); - Map partsMap = nap.get2(); + final long reqId = qryIdGen.incrementAndGet(); - DistributedUpdateRun r = new DistributedUpdateRun(nodes.size()); + final GridRunningQueryInfo qryInfo = new GridRunningQueryInfo(reqId, selectQry, GridCacheQueryType.SQL_FIELDS, + schemaName, U.currentTimeMillis(), cancel, false); - final long reqId = qryIdGen.incrementAndGet(); + final Collection nodes = nodesParts.nodes(); + + final DistributedUpdateRun r = new DistributedUpdateRun(nodes.size(), qryInfo); GridH2DmlRequest req = new GridH2DmlRequest() .requestId(reqId) @@ -894,23 +882,40 @@ public long update( .pageSize(pageSize) .parameters(params) .timeout(timeoutMillis) + .partitions(convert(nodesParts.partitionsMap())) .flags(enforceJoinOrder ? GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER : 0); updRuns.put(reqId, r); + boolean release = false; + try { - if (send(nodes, req, parts == null ? null : new ExplicitPartitionsSpecializer(partsMap), false)) - awaitAllReplies(r, nodes, cancel); + ExplicitPartitionsSpecializer partsSpec = (parts == null) ? null : + new ExplicitPartitionsSpecializer(nodesParts.queryPartitionsMap()); + + cancel.set(new Runnable() { + @Override public void run() { + r.future().onCancelled(); + + send(nodes, new GridQueryCancelRequest(reqId), null, false); + } + }); + + if (send(nodes, req, partsSpec, false)) + return r.future().get(); - return r.updateCounter(); + throw new CacheException("Failed to send update request."); } catch (IgniteCheckedException | RuntimeException e) { + release = true; + log.error("Error in update: " + e.toString()); throw new CacheException("Failed to run update: ", e); } finally { - //TODO: cancel remote updates + if (release) + send(nodes, new GridQueryCancelRequest(reqId), null, false); if (!updRuns.remove(reqId, r)) U.warn(log, "Update run was already removed: " + reqId); @@ -924,11 +929,8 @@ private void onDmlResponse(final ClusterNode node, GridH2DmlResponse msg) { DistributedUpdateRun r = updRuns.get(reqId); - if (r == null) { - log.error("Unknown dml request with id " + reqId); - + if (r == null) return; - } if (r.handleResponse(node.id(), msg)) updRuns.remove(reqId); @@ -1017,25 +1019,6 @@ private void awaitAllReplies(ReduceQueryRun r, Collection nodes, Gr } } - /** */ - private void awaitAllReplies(DistributedUpdateRun r, Collection nodes, GridQueryCancel cancel) - throws IgniteInterruptedCheckedException, QueryCancelledException { - while (!U.await(r.latch(), 500, TimeUnit.MILLISECONDS)) { - - cancel.checkCancelled(); - - for (ClusterNode node : nodes) { - if (!ctx.discovery().alive(node)) { - r.handleNodeLeft(node.id()); - - assert r.latch().getCount() == 0; - - return; - } - } - } - } - /** * Gets or creates new fake table for index. * @@ -1421,10 +1404,17 @@ private static Map convert(Map m) { return res; } - // TODO: reuse this function in query() - /** */ - private IgniteBiTuple, Map> nodesForPartitions( - List cacheIds, AffinityTopologyVersion topVer, int[] parts, boolean isReplicatedOnly) { + /** + * Evaluates nodes and nodes to partitions map given a list of cache ids, topology version and partitions. + * + * @param cacheIds Cache ids. + * @param topVer Topology version. + * @param parts Paritions array. + * @param isReplicatedOnly Allow only replicated caches. + * @return Result. + */ + private NodesForPartitionsResult nodesForPartitions(List cacheIds, AffinityTopologyVersion topVer, + int[] parts, boolean isReplicatedOnly) { Collection nodes = null; Map partsMap = null; Map qryMap = null; @@ -1449,7 +1439,7 @@ private IgniteBiTuple, Map> nodes nodes = qryMap.keySet(); } - return new IgniteBiTuple<>(nodes, qryMap == null ? partsMap : qryMap); + return new NodesForPartitionsResult(nodes, partsMap, qryMap); } /** @@ -1568,6 +1558,11 @@ public Collection longRunningQueries(long duration) { res.add(run.queryInfo()); } + for (DistributedUpdateRun upd: updRuns.values()) { + if (upd.queryInfo().longQuery(curTime, duration)) + res.add(upd.queryInfo()); + } + return res; } @@ -1582,9 +1577,13 @@ public void cancelQueries(Collection queries) { if (run != null) run.queryInfo().cancel(); - } + else { + DistributedUpdateRun upd = updRuns.get(qryId); - // TODO: do we need updRuns processed here (and in longRunningQueries) as well? + if (upd != null) + upd.queryInfo().cancel(); + } + } } /** */ @@ -1644,4 +1643,47 @@ public ExplicitPartitionsSpecializer(Map partsMap) { return msg; } } + + /** + * Result of nodes to partitions mapping for a query or update. + */ + static class NodesForPartitionsResult { + /** */ + final Collection nodes; + + /** */ + final Map partsMap; + + /** */ + final Map qryMap; + + /** */ + NodesForPartitionsResult(Collection nodes, Map partsMap, + Map qryMap) { + this.nodes = nodes; + this.partsMap = partsMap; + this.qryMap = qryMap; + } + + /** + * @return Collection of nodes a message shall be sent to. + */ + Collection nodes() { + return nodes; + } + + /** + * @return Maps a node to partition array. + */ + Map partitionsMap() { + return partsMap; + } + + /** + * @return Maps a node to partition array. + */ + Map queryPartitionsMap() { + return qryMap; + } + } } \ No newline at end of file From f90adfad9528972a6a2bd262ffddb53824d55dd9 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Tue, 22 Aug 2017 12:54:52 +0300 Subject: [PATCH 03/40] IGNITE-6024: fixing todos --- .../query/h2/DmlStatementsProcessor.java | 82 +++++++------------ .../processors/query/h2/IgniteH2Indexing.java | 6 +- .../h2/twostep/GridMapQueryExecutor.java | 18 ++-- .../query/h2/twostep/MapNodeResults.java | 29 +++++++ 4 files changed, 76 insertions(+), 59 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 254948f2587e3..87bb2139f1a6d 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -434,59 +434,31 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo int pageSize = loc ? 0 : fieldsQry.getPageSize(); - switch (plan.mode) { - case MERGE: - return new UpdateResult(doMerge(plan, cur, pageSize), X.EMPTY_OBJECT_ARRAY); - - case INSERT: - return new UpdateResult(doInsert(plan, cur, pageSize), X.EMPTY_OBJECT_ARRAY); - - case UPDATE: - return doUpdate(plan, cur, pageSize); - - case DELETE: - return doDelete(cctx, cur, pageSize); - - default: - throw new IgniteSQLException("Unexpected DML operation [mode=" + plan.mode + ']', - IgniteQueryErrorCode.UNEXPECTED_OPERATION); - } + return processDmlSelectResult(cctx, plan, cur, pageSize); } - /** */ - private UpdateResult executeLocalDmlSelect(GridCacheContext cctx, String schemaName, UpdatePlan plan, - int pageSize, boolean isEnforceJoinOrder, int timeoutMillis, IndexingQueryFilter filters, - GridQueryCancel cancel, Object[] args) throws IgniteCheckedException { - - // TODO: similar code in executeUpdateStatement(), need to refactor - - final GridQueryFieldsResult res = idx.queryLocalSqlFields(schemaName, plan.selectQry, - F.asList(args), filters, isEnforceJoinOrder, timeoutMillis, cancel); - - QueryCursorImpl> cur = new QueryCursorImpl<>(new Iterable>() { - @Override public Iterator> iterator() { - try { - return new GridQueryCacheObjectsIterator(res.iterator(), idx.objectContext(), true); - } - catch (IgniteCheckedException e) { - throw new IgniteException(e); - } - } - }, cancel); - - + /** + * @param cctx Cache context. + * @param plan Update plan. + * @param cursor Cursor over select results. + * @param pageSize Page size. + * @return Pair [number of successfully processed items; keys that have failed to be processed] + * @throws IgniteCheckedException if failed. + */ + private UpdateResult processDmlSelectResult(GridCacheContext cctx, UpdatePlan plan, Iterable> cursor, + int pageSize) throws IgniteCheckedException { switch (plan.mode) { case MERGE: - return new UpdateResult(doMerge(plan, cur, pageSize), X.EMPTY_OBJECT_ARRAY); + return new UpdateResult(doMerge(plan, cursor, pageSize), X.EMPTY_OBJECT_ARRAY); case INSERT: - return new UpdateResult(doInsert(plan, cur, pageSize), X.EMPTY_OBJECT_ARRAY); + return new UpdateResult(doInsert(plan, cursor, pageSize), X.EMPTY_OBJECT_ARRAY); case UPDATE: - return doUpdate(plan, cur, pageSize); + return doUpdate(plan, cursor, pageSize); case DELETE: - return doDelete(cctx, cur, pageSize); + return doDelete(cctx, cursor, pageSize); default: throw new IgniteSQLException("Unexpected DML operation [mode=" + plan.mode + ']', @@ -1049,7 +1021,7 @@ private static PageProcessingResult processPage(GridCacheContext cctx, /** */ UpdateResult mapDistributedUpdate(byte mode, String schemaName, String targetTable, String[] colNames, String qry, Object[] params, int pageSize, int timeoutMillis, - IndexingQueryFilter filter) throws IgniteCheckedException { + IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException { GridH2Table tgtTbl = idx.dataTable(schemaName, targetTable); //TODO: use QueryTable instead UpdateMode[] modes = UpdateMode.values(); @@ -1064,7 +1036,6 @@ UpdateResult mapDistributedUpdate(byte mode, String schemaName, String targetTab colNames, qry); boolean isEnforceJoinOrder = false; // TODO fix this. - GridQueryCancel cancel = new GridQueryCancel(); CacheOperationContext opCtx = cctx.operationContextPerCall(); @@ -1082,17 +1053,26 @@ else if (!opCtx.isKeepBinary()) cctx.operationContextPerCall(newOpCtx); } - UpdateResult res; - try { - res = executeLocalDmlSelect(cctx, schemaName, plan, pageSize, isEnforceJoinOrder, - timeoutMillis, filter, cancel, params); + final GridQueryFieldsResult res = idx.queryLocalSqlFields(schemaName, plan.selectQry, + F.asList(params), filter, isEnforceJoinOrder, timeoutMillis, cancel); + + QueryCursorImpl> cur = new QueryCursorImpl<>(new Iterable>() { + @Override public Iterator> iterator() { + try { + return new GridQueryCacheObjectsIterator(res.iterator(), idx.objectContext(), true); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + }, cancel); + + return processDmlSelectResult(cctx, plan, cur, pageSize); } finally { cctx.operationContextPerCall(opCtx); } - - return res; } /** */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 1d3858af777f7..573b74639a5ca 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1442,10 +1442,10 @@ UpdateResult runDistributedUpdate( /** */ public UpdateResult mapDistributedUpdate(byte mode, String schemaName, String targetTable, String[] colNames, - String qry, Object[] params, int pageSize, int timeoutMillis, IndexingQueryFilter filter) - throws IgniteCheckedException { + String qry, Object[] params, int pageSize, int timeoutMillis, IndexingQueryFilter filter, + GridQueryCancel cancel) throws IgniteCheckedException { return dmlProc.mapDistributedUpdate(mode, schemaName, targetTable, colNames, qry, params, pageSize, - timeoutMillis, filter); + timeoutMillis, filter, cancel); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 8019536798603..c82f84fb4f8ca 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -54,6 +54,7 @@ import org.apache.ignite.internal.processors.cache.query.CacheQueryType; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable; import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; +import org.apache.ignite.internal.processors.query.GridQueryCancel; import org.apache.ignite.internal.processors.query.h2.H2Utils; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; import org.apache.ignite.internal.processors.query.h2.UpdateResult; @@ -752,6 +753,8 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th final List cacheIds = req.caches(); + long reqId = req.requestId(); + AffinityTopologyVersion topVer = req.topologyVersion(); List reserved = new ArrayList<>(); @@ -759,26 +762,31 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th if (!reservePartitions(cacheIds, topVer, parts, reserved)) { log.error("Failed to reserve partitions for DML request"); - sendUpdateResponse(node, req.requestId(), null, "Failed to reserve partitions for DML request"); + sendUpdateResponse(node, reqId, null, "Failed to reserve partitions for DML request"); return; } + MapNodeResults nodeResults = resultsForNode(node.id()); + try { String query = req.queries().get(0).query(); IndexingQueryFilter filter = h2.backupFilter(req.topologyVersion(), req.queryPartitions()); - UpdateResult updRes = h2.mapDistributedUpdate(req.mode(), req.schemaName(), req.targetTable(), - req.columnNames(), query, req.parameters(), req.pageSize(), req.timeout(), filter); + GridQueryCancel cancel = nodeResults.putUpdate(reqId); - sendUpdateResponse(node, req.requestId(), updRes, null); + UpdateResult updRes = h2.mapDistributedUpdate(req.mode(), req.schemaName(), req.targetTable(), + req.columnNames(), query, req.parameters(), req.pageSize(), req.timeout(), filter, cancel); + sendUpdateResponse(node, reqId, updRes, null); } catch (Exception e) { log.error("Error processing dml request " + e.toString()); - sendUpdateResponse(node, req.requestId(), null, e.getMessage()); + nodeResults.removeUpdate(reqId); + + sendUpdateResponse(node, reqId, null, e.getMessage()); } finally { if (!F.isEmpty(reserved)) { diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapNodeResults.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapNodeResults.java index 2d20c8d52593e..3fcbcee84dc4b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapNodeResults.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapNodeResults.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.query.h2.twostep; +import org.apache.ignite.internal.processors.query.GridQueryCancel; import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap; import org.jsr166.ConcurrentHashMap8; @@ -32,6 +33,9 @@ class MapNodeResults { /** */ private final ConcurrentMap res = new ConcurrentHashMap8<>(); + /** Cancel state for update requests. */ + private final ConcurrentMap updCancels = new ConcurrentHashMap8<>(); + /** */ private final GridBoundedConcurrentLinkedHashMap qryHist = new GridBoundedConcurrentLinkedHashMap<>(1024, 1024, 0.75f, 64, PER_SEGMENT_Q); @@ -88,6 +92,12 @@ public void cancelRequest(long reqId) { removed.cancel(true); } } + + // Cancel update request + GridQueryCancel updCancel = updCancels.remove(reqId); + + if (updCancel != null) + updCancel.cancel(); } /** @@ -110,12 +120,31 @@ public MapQueryResults put(long reqId, int segmentId, MapQueryResults qr) { return res.put(new MapRequestKey(nodeId, reqId, segmentId), qr); } + /** + * @param reqId Request id. + * @return Cancel state. + */ + public GridQueryCancel putUpdate(long reqId) { + return updCancels.put(reqId, new GridQueryCancel()); + } + + /** + * @param reqId Request id. + */ + public void removeUpdate(long reqId) { + updCancels.remove(reqId); + } + /** * Cancel all node queries. */ public void cancelAll() { for (MapQueryResults ress : res.values()) ress.cancel(true); + + // Cancel update requests + for (GridQueryCancel upd: updCancels.values()) + upd.cancel(); } } From 0ecdf75dde64b0aab365996c50fa41852ec3ea2a Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Tue, 22 Aug 2017 13:50:20 +0300 Subject: [PATCH 04/40] IGNITE-6024: fixed javadocs --- .../query/h2/DmlStatementsProcessor.java | 36 +++++++++++++---- .../processors/query/h2/IgniteH2Indexing.java | 39 +++++++++++++++++-- .../processors/query/h2/dml/UpdatePlan.java | 2 +- .../query/h2/dml/UpdatePlanBuilder.java | 14 ++++++- .../h2/twostep/DistributedUpdateRun.java | 34 ++++++++++++---- .../h2/twostep/GridMapQueryExecutor.java | 15 +++---- .../h2/twostep/GridReduceQueryExecutor.java | 26 +++++++++++-- .../h2/twostep/msg/GridH2DmlRequest.java | 12 +++--- .../h2/twostep/msg/GridH2DmlResponse.java | 28 ++++++++----- 9 files changed, 160 insertions(+), 46 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 87bb2139f1a6d..b743667239101 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -215,12 +215,18 @@ else if (items == 0L) return new UpdateResult(items, errKeys); } - /** */ - private boolean checkDistributed(String schemaName, UpdatePlan p, SqlFieldsQuery fieldsQry) { + /** + * + * @param schemaName Schema name. + * @param plan Update plan. + * @param fieldsQry Initial update query. + * @return {@code true} if query can be optimized with distributed DML. + */ + private boolean checkDistributed(String schemaName, UpdatePlan plan, SqlFieldsQuery fieldsQry) { Connection c = idx.connectionForSchema(schemaName); try { - PreparedStatement stmt = c.prepareStatement(p.selectQry); + PreparedStatement stmt = c.prepareStatement(plan.selectQry); idx.bindParameters(stmt, F.asList(fieldsQry.getArgs())); @@ -231,16 +237,16 @@ private boolean checkDistributed(String schemaName, UpdatePlan p, SqlFieldsQuery fieldsQry.isEnforceJoinOrder(), idx); // TODO: find a way to set aside sub-queries - p.distributed = !qry.isReplicatedOnly() && qry.mapQueries().size() == 1 && qry.skipMergeTable(); + plan.distributed = !qry.isReplicatedOnly() && qry.mapQueries().size() == 1 && qry.skipMergeTable(); - if (p.distributed) - p.cacheIds = idx.collectCacheIds(CU.cacheId(p.tbl.cacheName()), qry); + if (plan.distributed) + plan.cacheIds = idx.collectCacheIds(CU.cacheId(plan.tbl.cacheName()), qry); } catch (SQLException | IgniteCheckedException e) { throw new IgniteException(e); } - return p.distributed; + return plan.distributed; } /** @@ -1018,7 +1024,21 @@ private static PageProcessingResult processPage(GridCacheContext cctx, return new IgniteBiTuple<>(key, val); } - /** */ + /** + * + * @param mode Update mode. + * @param schemaName Schema name. + * @param targetTable Target table. + * @param colNames Column names. + * @param qry Query. + * @param params Query parameters. + * @param pageSize Page size. + * @param timeoutMillis Timeout. + * @param filter Filter. + * @param cancel Cancel state. + * @return Update result. + * @throws IgniteCheckedException if failed. + */ UpdateResult mapDistributedUpdate(byte mode, String schemaName, String targetTable, String[] colNames, String qry, Object[] params, int pageSize, int timeoutMillis, IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException { diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 9594a898ee959..dba5b97aa8ff1 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1192,7 +1192,19 @@ private Iterable> runQueryTwoStep( }; } - /** */ + /** + * Run DML on remote nodes. + * + * @param schemaName Schema name. + * @param fieldsQry Initial update query. + * @param cacheIds Cache identifiers. + * @param updateMode Update mode. + * @param tgtTable Target table. + * @param colNames Column names. + * @param selectQry Select query derived from initial update query. + * @param cancel Cancel state. + * @return Update result. + */ UpdateResult runDistributedUpdate( String schemaName, SqlFieldsQuery fieldsQry, @@ -1440,7 +1452,22 @@ UpdateResult runDistributedUpdate( return cursor; } - /** */ + /** + * Run DML request from other node. + * + * @param mode Update mode. + * @param schemaName Schema name. + * @param targetTable Target table. + * @param colNames Column names. + * @param qry Query. + * @param params SQL parameters. + * @param pageSize Page size. + * @param timeoutMillis Timeout. + * @param filter Filter. + * @param cancel Cancel state. + * @return Update result. + * @throws IgniteCheckedException if failed. + */ public UpdateResult mapDistributedUpdate(byte mode, String schemaName, String targetTable, String[] colNames, String qry, Object[] params, int pageSize, int timeoutMillis, IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException { @@ -2400,7 +2427,13 @@ private int bindPartitionInfoParameter(CacheQueryPartitionInfo partInfo, Object[ U.close(conn, log); } - /** */ + /** + * Collect cache identifiers from two-step query. + * + * @param mainCacheId Id of main cache. + * @param twoStepQry Two-step query. + * @return Result. + */ public List collectCacheIds(@Nullable Integer mainCacheId, GridCacheTwoStepQuery twoStepQry) { LinkedHashSet caches0 = new LinkedHashSet<>(); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 747756993e525..a0fa409380290 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -68,7 +68,7 @@ public final class UpdatePlan { /** Distributed update flag. */ public boolean distributed = false; - /** */ + /** Integer cache identifiers. */ public List cacheIds; /** */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java index 2f9fcd6633615..18ed88d54aeea 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java @@ -47,6 +47,7 @@ import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlUnion; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlUpdate; +import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlRequest; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; @@ -300,7 +301,18 @@ else if (stmt instanceof GridSqlDelete) { } } - /** */ + /** + * Recover update plan from parameters received in {@link GridH2DmlRequest message} + * + * @param cctx Cache context. + * @param updateMode Update mode. + * @param tgtTbl Target table. + * @param desc Target table descriptor. + * @param colNames Updated column names. + * @param qry SQL select for update. + * @return Update plan. + * @throws IgniteCheckedException If failed. + */ public static UpdatePlan planFromMessage(GridCacheContext cctx, UpdateMode updateMode, GridH2Table tgtTbl, GridH2RowDescriptor desc, String[] colNames, String qry) throws IgniteCheckedException { boolean hasKeyProps = false; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java index f48f88d11640f..e7c545f3de704 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java @@ -29,7 +29,7 @@ import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.typedef.F; -/** */ +/** Context for DML operation on reducer node. */ class DistributedUpdateRun { /** Response counter. */ private AtomicLong rspCntr; @@ -46,7 +46,12 @@ class DistributedUpdateRun { /** Result future. */ private GridFutureAdapter fut = new GridFutureAdapter<>(); - /** */ + /** + * Constructor. + * + * @param nodeCount Number of nodes to await results from. + * @param qry Query info. + */ DistributedUpdateRun(int nodeCount, GridRunningQueryInfo qry) { rspCntr = new AtomicLong(nodeCount); @@ -67,17 +72,30 @@ GridFutureAdapter future() { return fut; } - /** */ + /** + * Handle disconnection. + * @param e Pre-formatted error. + */ void handleDisconnect(CacheException e) { fut.onDone(new IgniteCheckedException("Update failed.", e)); } - /** */ + /** + * Handle leave of a node. + * + * @param nodeId Node id. + */ void handleNodeLeft(UUID nodeId) { fut.onDone(new IgniteCheckedException("Update failed: node " + nodeId + " has left")); } - /** */ + /** + * Handle response from remote node. + * + * @param id Node id. + * @param msg Response message. + * @return {@code true} if no more responses are expected. + */ boolean handleResponse(UUID id, GridH2DmlResponse msg) { boolean done = rspCntr.decrementAndGet() == 0; @@ -100,12 +118,14 @@ boolean handleResponse(UUID id, GridH2DmlResponse msg) { case GridH2DmlResponse.STATUS_ERROR: String err = msg.error(); - fut.onDone(new IgniteCheckedException("Update failed. " + (F.isEmpty(err)? "" : err))); + fut.onDone(new IgniteCheckedException("Update failed. " + (F.isEmpty(err)? "" : err) + "[reqId=" + + msg.requestId() + ", node=" + id + "].")); return false; default: - fut.onDone(new IgniteCheckedException("Update failed. Invalid status in response message.")); + fut.onDone(new IgniteCheckedException("Update failed. Invalid status in response message [reqId=" + + msg.requestId() + ", node=" + id + "].")); return false; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index c82f84fb4f8ca..7e6ec9f5e813c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -822,19 +822,20 @@ private void sendError(ClusterNode node, long qryReqId, Throwable err) { } /** + * Sends update response for DML request. * - * @param node - * @param reqId - * @param updResult - * @param error + * @param node Node. + * @param reqId Request id. + * @param updResult Update result. + * @param error Error message. */ private void sendUpdateResponse(ClusterNode node, long reqId, UpdateResult updResult, String error) { try { - byte status = (error != null) ? GridH2DmlResponse.STATUS_ERROR : + byte status = (error != null || updResult == null) ? GridH2DmlResponse.STATUS_ERROR : F.isEmpty(updResult.errorKeys()) ? GridH2DmlResponse.STATUS_OK : GridH2DmlResponse.STATUS_ERR_KEYS; - GridH2DmlResponse rsp = new GridH2DmlResponse(reqId, status, updResult.counter(), updResult.errorKeys(), - error); + GridH2DmlResponse rsp = new GridH2DmlResponse(reqId, status, updResult == null ? 0: updResult.counter(), + updResult == null ? null: updResult.errorKeys(), error); if (node.isLocal()) h2.reduceQueryExecutor().onMessage(ctx.localNodeId(), rsp); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index 680a373176338..b6d1bb228ac8e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -134,7 +134,7 @@ public class GridReduceQueryExecutor { /** */ private final ConcurrentMap runs = new ConcurrentHashMap8<>(); - /** */ + /** Contexts of running DML requests. */ private final ConcurrentMap updRuns = new ConcurrentHashMap8<>(); /** */ @@ -843,7 +843,22 @@ public Iterator> query( } } - /** */ + /** + * + * @param schemaName Schema name. + * @param cacheIds Cache ids. + * @param updateMode Update mode. + * @param tgtTable Target table. + * @param colNames Column names. + * @param selectQry Select query. + * @param enforceJoinOrder Enforce join order of tables. + * @param pageSize Page size. + * @param timeoutMillis Timeout. + * @param params SQL parameters. + * @param parts Partitions. + * @param cancel Cancel state. + * @return Update result. + */ public UpdateResult update( String schemaName, List cacheIds, @@ -922,7 +937,12 @@ public UpdateResult update( } } - /** */ + /** + * Process response for DML request. + * + * @param node Node. + * @param msg Message. + */ private void onDmlResponse(final ClusterNode node, GridH2DmlResponse msg) { try { long reqId = msg.requestId(); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java index e878767576b71..91fb6f9288160 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java @@ -44,15 +44,15 @@ import static org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery.EMPTY_PARAMS; -/** */ -public class GridH2DmlRequest implements Message, GridCacheQueryMarshallable { +/** Request for DML operation on remote node. */ +public class GridH2DmlRequest implements Message, GridCacheQueryMarshallable { /** */ private static final long serialVersionUID = 0L; - /** */ + /** Request id. */ private long reqId; - /** */ + /** Cache identifiers. */ @GridToStringInclude @GridDirectCollection(Integer.class) private List caches; @@ -69,10 +69,10 @@ public class GridH2DmlRequest implements Message, GridCacheQueryMarshallable { @GridToStringInclude private int[] qryParts; - /** */ + /** Page size. */ private int pageSize; - /** */ + /** Query. */ @GridToStringInclude @GridDirectCollection(Message.class) private List qrys; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java index 02e98db46e709..3e218a892382c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java @@ -24,33 +24,33 @@ import org.apache.ignite.plugin.extensions.communication.MessageReader; import org.apache.ignite.plugin.extensions.communication.MessageWriter; -/** */ +/** Response to remote DML request. */ public class GridH2DmlResponse implements Message { /** */ private static final long serialVersionUID = 0L; - /** */ + /** Successful execution. */ public static final byte STATUS_OK = 0; - /** */ + /** Error occurred. */ public static final byte STATUS_ERROR = 1; - /** */ + /** Partial success. Erroneous keys identified. */ public static final byte STATUS_ERR_KEYS = 2; - /** */ + /** Request id. */ private long reqId; - /** */ + /** Status. */ private byte status; - /** */ + /** Updated rows counter. */ private long updCnt; - /** */ + /** Error message. */ private String err; - /** */ + /** Keys that failed. */ private Object[] errKeys; /** @@ -60,7 +60,15 @@ public GridH2DmlResponse() { // No-op. } - /** */ + /** + * Constructor. + * + * @param reqId Request id. + * @param status Status. + * @param updCnt Updated row number. + * @param errKeys Erroneous keys. + * @param error Error message. + */ public GridH2DmlResponse(long reqId, byte status, long updCnt, Object[] errKeys, String error) { this.reqId = reqId; this.status = status; From d09f88c19dc7d6abb175761f1d76e3a36f66c7d8 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Tue, 22 Aug 2017 18:37:07 +0300 Subject: [PATCH 05/40] IGNITE-6024: some fixes --- .../query/h2/DmlStatementsProcessor.java | 46 ++++++++++--------- .../h2/twostep/DistributedUpdateRun.java | 2 - 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index b743667239101..2175b61476cdc 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -40,6 +40,7 @@ import javax.cache.processor.EntryProcessorException; import javax.cache.processor.EntryProcessorResult; import javax.cache.processor.MutableEntry; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteDataStreamer; @@ -216,37 +217,38 @@ else if (items == 0L) } /** + * Checks whether the given update plan can be distributed and updates it accordingly. * - * @param schemaName Schema name. - * @param plan Update plan. * @param fieldsQry Initial update query. - * @return {@code true} if query can be optimized with distributed DML. + * @param prepStmt Prepared statement for initial update query. + * @param plan Update plan. + * @throws IgniteCheckedException if failed. */ - private boolean checkDistributed(String schemaName, UpdatePlan plan, SqlFieldsQuery fieldsQry) { - Connection c = idx.connectionForSchema(schemaName); - + private void checkPlanCanBeDistributed(SqlFieldsQuery fieldsQry, PreparedStatement prepStmt, UpdatePlan plan) + throws IgniteCheckedException { try { - PreparedStatement stmt = c.prepareStatement(plan.selectQry); + Connection conn = prepStmt.getConnection(); - idx.bindParameters(stmt, F.asList(fieldsQry.getArgs())); + // Get a new prepared statement for derived select query. + try (PreparedStatement stmt = conn.prepareStatement(plan.selectQry)) { + idx.bindParameters(stmt, F.asList(fieldsQry.getArgs())); - GridCacheTwoStepQuery qry = GridSqlQuerySplitter.split((JdbcPreparedStatement)stmt, - fieldsQry.getArgs(), - fieldsQry.isCollocated(), - fieldsQry.isDistributedJoins(), - fieldsQry.isEnforceJoinOrder(), idx); + GridCacheTwoStepQuery qry = GridSqlQuerySplitter.split((JdbcPreparedStatement)stmt, + fieldsQry.getArgs(), + fieldsQry.isCollocated(), + fieldsQry.isDistributedJoins(), + fieldsQry.isEnforceJoinOrder(), idx); - // TODO: find a way to set aside sub-queries - plan.distributed = !qry.isReplicatedOnly() && qry.mapQueries().size() == 1 && qry.skipMergeTable(); + // TODO: find a way to set aside sub-queries + plan.distributed = !qry.isReplicatedOnly() && qry.mapQueries().size() == 1 && qry.skipMergeTable(); - if (plan.distributed) - plan.cacheIds = idx.collectCacheIds(CU.cacheId(plan.tbl.cacheName()), qry); + if (plan.distributed) + plan.cacheIds = idx.collectCacheIds(CU.cacheId(plan.tbl.cacheName()), qry); + } } - catch (SQLException | IgniteCheckedException e) { - throw new IgniteException(e); + catch (SQLException e) { + throw new IgniteCheckedException(e); } - - return plan.distributed; } /** @@ -495,7 +497,7 @@ private UpdatePlan getPlanForStatement(String schema, PreparedStatement prepStmt res = UpdatePlanBuilder.planForStatement(p, errKeysPos); if (!loc && !F.isEmpty(res.selectQry)) - checkDistributed(schema, res, fieldsQry); + checkPlanCanBeDistributed(fieldsQry, prepStmt, res); // Don't cache re-runs if (errKeysPos == null) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java index e7c545f3de704..25c946b34b544 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java @@ -101,8 +101,6 @@ boolean handleResponse(UUID id, GridH2DmlResponse msg) { switch (msg.status()) { case GridH2DmlResponse.STATUS_ERR_KEYS: - updCntr.addAndGet(msg.updateCounter()); - if (!F.isEmpty(msg.errorKeys())) errorKeys.addAll(Arrays.asList(msg.errorKeys())); From 60bd615bc009c1a207aec25108849e6192e183ae Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Wed, 23 Aug 2017 13:59:25 +0300 Subject: [PATCH 06/40] IGNITE-6024: Added sub-queries check --- .../cache/query/GridCacheSqlQuery.java | 23 ++++++++++++++++ .../query/h2/DmlStatementsProcessor.java | 6 +++-- .../query/h2/sql/GridSqlQuerySplitter.java | 26 +++++++++++++++++++ .../h2/twostep/GridMapQueryExecutor.java | 4 +-- 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java index d3746f30cbf0c..30125869e8970 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java @@ -74,6 +74,11 @@ public class GridCacheSqlQuery implements Message { @GridDirectTransient private transient Object[] derivedPartitions; + /** Flag indicating that query contains sub-queries. */ + @GridToStringInclude + @GridDirectTransient + private transient boolean hasSubQries; + /** * For {@link Message}. */ @@ -259,6 +264,7 @@ public GridCacheSqlQuery copy() { cp.sort = sort; cp.partitioned = partitioned; cp.derivedPartitions = derivedPartitions; + cp.hasSubQries = hasSubQries; return cp; } @@ -347,4 +353,21 @@ public GridCacheSqlQuery derivedPartitions(Object[] derivedPartitions) { return this; } + + /** + * @return {@code true} if query contains sub-queries. + */ + public boolean hasSubQueries() { + return hasSubQries; + } + + /** + * @param hasSubQries Flag indicating that query contains sub-queries. + * @return {@code this}. + */ + public GridCacheSqlQuery hasSubQueries(boolean hasSubQries) { + this.hasSubQries = hasSubQries; + + return this; + } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 2175b61476cdc..71c2ed0ed62c6 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -239,8 +239,10 @@ private void checkPlanCanBeDistributed(SqlFieldsQuery fieldsQry, PreparedStateme fieldsQry.isDistributedJoins(), fieldsQry.isEnforceJoinOrder(), idx); - // TODO: find a way to set aside sub-queries - plan.distributed = !qry.isReplicatedOnly() && qry.mapQueries().size() == 1 && qry.skipMergeTable(); + plan.distributed = !qry.isReplicatedOnly() && + qry.skipMergeTable() && + qry.mapQueries().size() == 1 && + !qry.mapQueries().get(0).hasSubQueries(); if (plan.distributed) plan.cacheIds = idx.collectCacheIds(CU.cacheId(plan.tbl.cacheName()), qry); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java index 1578f9fb78dc0..e1b8e5836b051 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java @@ -1511,6 +1511,12 @@ private void splitSelect( rdcQry.distinct(true); } + // -- SUB-QUERIES + boolean hasSubQueries = hasSubQueries(mapQry.where()); + + for (int i = 0; i < mapQry.columns(false).size(); i++) + hasSubQueries |= hasSubQueries(mapQry.column(i)); + // Replace the given select with generated reduce query in the parent. prnt.child(childIdx, rdcQry); @@ -1521,6 +1527,7 @@ private void splitSelect( map.columns(collectColumns(mapExps)); map.sortColumns(mapQry.sort()); map.partitioned(hasPartitionedTables(mapQry)); + map.hasSubQueries(hasSubQueries); if (map.isPartitioned()) map.derivedPartitions(derivePartitionsFromQuery(mapQry, ctx)); @@ -1544,6 +1551,25 @@ private static boolean hasPartitionedTables(GridSqlAst ast) { return false; } + /** + * @param ast Map query AST. + * @return {@code true} If the given AST has sub-queries. + */ + private boolean hasSubQueries(GridSqlAst ast) { + if (ast == null) + return false; + + if (ast instanceof GridSqlSubquery) + return true; + + for (int childIdx = 0; childIdx < ast.size(); childIdx++) { + if (hasSubQueries(ast.child(childIdx))) + return true; + } + + return false; + } + /** * @param sqlQry Query. * @param qryAst Select AST. diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 7e6ec9f5e813c..1729a72f70558 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -834,8 +834,8 @@ private void sendUpdateResponse(ClusterNode node, long reqId, UpdateResult updRe byte status = (error != null || updResult == null) ? GridH2DmlResponse.STATUS_ERROR : F.isEmpty(updResult.errorKeys()) ? GridH2DmlResponse.STATUS_OK : GridH2DmlResponse.STATUS_ERR_KEYS; - GridH2DmlResponse rsp = new GridH2DmlResponse(reqId, status, updResult == null ? 0: updResult.counter(), - updResult == null ? null: updResult.errorKeys(), error); + GridH2DmlResponse rsp = new GridH2DmlResponse(reqId, status, updResult == null ? 0 : updResult.counter(), + updResult == null ? null : updResult.errorKeys(), error); if (node.isLocal()) h2.reduceQueryExecutor().onMessage(ctx.localNodeId(), rsp); From 0b149ee7f6bc676d7924af6ba2744ab826c429fb Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Wed, 23 Aug 2017 16:24:52 +0300 Subject: [PATCH 07/40] IGNITE-6024: simplified - no update plan passing over net --- .../query/h2/DmlStatementsProcessor.java | 71 +------ .../processors/query/h2/IgniteH2Indexing.java | 34 ++- .../query/h2/dml/UpdatePlanBuilder.java | 71 ------- .../h2/twostep/GridMapQueryExecutor.java | 19 +- .../h2/twostep/GridReduceQueryExecutor.java | 15 +- .../h2/twostep/msg/GridH2DmlRequest.java | 200 +++--------------- .../h2/twostep/msg/GridH2DmlResponse.java | 53 ++++- 7 files changed, 112 insertions(+), 351 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 71c2ed0ed62c6..8aa00fd79278a 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -40,7 +40,6 @@ import javax.cache.processor.EntryProcessorException; import javax.cache.processor.EntryProcessorResult; import javax.cache.processor.MutableEntry; -import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteDataStreamer; @@ -71,7 +70,6 @@ import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlan; import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlanBuilder; import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; -import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter; import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap; @@ -405,7 +403,7 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo return doFastUpdate(plan, fieldsQry.getArgs()); } - if (plan.distributed) + if (plan.distributed && !loc) return doDistributedUpdate(schemaName, fieldsQry, plan, cancel); assert !F.isEmpty(plan.selectQry); @@ -554,8 +552,7 @@ private UpdateResult doDistributedUpdate(String schemaName, SqlFieldsQuery field if (cancel == null) cancel = new GridQueryCancel(); - return idx.runDistributedUpdate(schemaName, fieldsQry, plan.cacheIds, (byte)plan.mode.ordinal(), - plan.tbl.getName(), plan.colNames, plan.selectQry, cancel); + return idx.runDistributedUpdate(schemaName, fieldsQry, plan.cacheIds, cancel); } /** @@ -1030,73 +1027,17 @@ private static PageProcessingResult processPage(GridCacheContext cctx, /** * - * @param mode Update mode. * @param schemaName Schema name. - * @param targetTable Target table. - * @param colNames Column names. - * @param qry Query. - * @param params Query parameters. - * @param pageSize Page size. - * @param timeoutMillis Timeout. + * @param stmt Prepared statement. + * @param fldsQry Query. * @param filter Filter. * @param cancel Cancel state. * @return Update result. * @throws IgniteCheckedException if failed. */ - UpdateResult mapDistributedUpdate(byte mode, String schemaName, String targetTable, - String[] colNames, String qry, Object[] params, int pageSize, int timeoutMillis, + UpdateResult mapDistributedUpdate(String schemaName, PreparedStatement stmt, SqlFieldsQuery fldsQry, IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException { - GridH2Table tgtTbl = idx.dataTable(schemaName, targetTable); //TODO: use QueryTable instead - - UpdateMode[] modes = UpdateMode.values(); - - assert mode >= 0 && mode < modes.length; - - UpdateMode updateMode = modes[mode]; - - GridCacheContext cctx = tgtTbl.rowDescriptor().context(); - - UpdatePlan plan = UpdatePlanBuilder.planFromMessage(cctx, updateMode, tgtTbl, tgtTbl.rowDescriptor(), - colNames, qry); - - boolean isEnforceJoinOrder = false; // TODO fix this. - - CacheOperationContext opCtx = cctx.operationContextPerCall(); - - // Force keepBinary for operation context to avoid binary deserialization inside entry processor - if (cctx.binaryMarshaller()) { - CacheOperationContext newOpCtx = null; - - if (opCtx == null) - // Mimics behavior of GridCacheAdapter#keepBinary and GridCacheProxyImpl#keepBinary - newOpCtx = new CacheOperationContext(false, null, true, null, false, null, false); - else if (!opCtx.isKeepBinary()) - newOpCtx = opCtx.keepBinary(); - - if (newOpCtx != null) - cctx.operationContextPerCall(newOpCtx); - } - - try { - final GridQueryFieldsResult res = idx.queryLocalSqlFields(schemaName, plan.selectQry, - F.asList(params), filter, isEnforceJoinOrder, timeoutMillis, cancel); - - QueryCursorImpl> cur = new QueryCursorImpl<>(new Iterable>() { - @Override public Iterator> iterator() { - try { - return new GridQueryCacheObjectsIterator(res.iterator(), idx.objectContext(), true); - } - catch (IgniteCheckedException e) { - throw new IgniteException(e); - } - } - }, cancel); - - return processDmlSelectResult(cctx, plan, cur, pageSize); - } - finally { - cctx.operationContextPerCall(opCtx); - } + return updateSqlFields(schemaName, stmt, fldsQry, true, filter, cancel); } /** */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index dba5b97aa8ff1..0273b6735c5bb 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1198,10 +1198,6 @@ private Iterable> runQueryTwoStep( * @param schemaName Schema name. * @param fieldsQry Initial update query. * @param cacheIds Cache identifiers. - * @param updateMode Update mode. - * @param tgtTable Target table. - * @param colNames Column names. - * @param selectQry Select query derived from initial update query. * @param cancel Cancel state. * @return Update result. */ @@ -1209,15 +1205,10 @@ UpdateResult runDistributedUpdate( String schemaName, SqlFieldsQuery fieldsQry, List cacheIds, - byte updateMode, - String tgtTable, - String[] colNames, - String selectQry, GridQueryCancel cancel) { - - return rdcQryExec.update(schemaName, cacheIds, updateMode, tgtTable, colNames, selectQry, + return rdcQryExec.update(schemaName, cacheIds, fieldsQry.getSql(), fieldsQry.getArgs(), fieldsQry.isEnforceJoinOrder(), fieldsQry.getPageSize(), fieldsQry.getTimeout(), - fieldsQry.getArgs(), fieldsQry.getPartitions(), cancel); + fieldsQry.getPartitions(), cancel); } /** {@inheritDoc} */ @@ -1455,24 +1446,23 @@ UpdateResult runDistributedUpdate( /** * Run DML request from other node. * - * @param mode Update mode. * @param schemaName Schema name. - * @param targetTable Target table. - * @param colNames Column names. - * @param qry Query. - * @param params SQL parameters. - * @param pageSize Page size. - * @param timeoutMillis Timeout. + * @param fldsQry Query. * @param filter Filter. * @param cancel Cancel state. * @return Update result. * @throws IgniteCheckedException if failed. */ - public UpdateResult mapDistributedUpdate(byte mode, String schemaName, String targetTable, String[] colNames, - String qry, Object[] params, int pageSize, int timeoutMillis, IndexingQueryFilter filter, + public UpdateResult mapDistributedUpdate(String schemaName, SqlFieldsQuery fldsQry, IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException { - return dmlProc.mapDistributedUpdate(mode, schemaName, targetTable, colNames, qry, params, pageSize, - timeoutMillis, filter, cancel); + Connection conn = connectionForSchema(schemaName); + + H2Utils.setupConnection(conn, false, fldsQry.isEnforceJoinOrder()); + + PreparedStatement stmt = preparedStatementWithParams(conn, fldsQry.getSql(), + Arrays.asList(fldsQry.getArgs()), true); + + return dmlProc.mapDistributedUpdate(schemaName, stmt, fldsQry, filter, cancel); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java index 18ed88d54aeea..32d5ad1bf279c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java @@ -47,7 +47,6 @@ import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlUnion; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlUpdate; -import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlRequest; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; @@ -301,76 +300,6 @@ else if (stmt instanceof GridSqlDelete) { } } - /** - * Recover update plan from parameters received in {@link GridH2DmlRequest message} - * - * @param cctx Cache context. - * @param updateMode Update mode. - * @param tgtTbl Target table. - * @param desc Target table descriptor. - * @param colNames Updated column names. - * @param qry SQL select for update. - * @return Update plan. - * @throws IgniteCheckedException If failed. - */ - public static UpdatePlan planFromMessage(GridCacheContext cctx, UpdateMode updateMode, GridH2Table tgtTbl, - GridH2RowDescriptor desc, String[] colNames, String qry) throws IgniteCheckedException { - boolean hasKeyProps = false; - boolean hasValProps = false; - int keyColIdx = -1; - int valColIdx = -1; - int[] colTypes = null; - - if (updateMode != UpdateMode.DELETE) { - colTypes = new int[colNames.length]; - - for (int i = 0; i < colTypes.length; ++i) { - String colName = colNames[i]; - - Column col = tgtTbl.getColumn(colName); - - colTypes[i] = col.getType(); - - int colId = col.getColumnId(); - - if (desc.isKeyColumn(colId)) { - keyColIdx = i; - - continue; - } - - if (desc.isValueColumn(colId)) { - valColIdx = i; - - continue; - } - - GridQueryProperty prop = desc.type().property(colName); - - assert prop != null : "Property '" + colName + "' not found."; - - if (prop.key()) - hasKeyProps = true; - else - hasValProps = true; - } - - if (updateMode == UpdateMode.UPDATE) - valColIdx = (valColIdx != -1) ? valColIdx + 2 : -1; - } - - KeyValueSupplier keySupplier = (updateMode == UpdateMode.INSERT || updateMode == UpdateMode.MERGE) ? - createSupplier(cctx, desc.type(), keyColIdx, hasKeyProps, true, false) : null; - - int valSupColIdx = (updateMode == UpdateMode.UPDATE)? ((valColIdx != -1) ? valColIdx : 1) : valColIdx; - - KeyValueSupplier valSupplier = (updateMode == UpdateMode.DELETE) ? null : - createSupplier(cctx, desc.type(), valSupColIdx, hasValProps, false, updateMode == UpdateMode.UPDATE); - - return UpdatePlan.fromMessage(updateMode, tgtTbl, colNames, colTypes, qry, keySupplier, valSupplier, - keyColIdx, valColIdx); - } - /** * Detect appropriate method of instantiating key or value (take from param, create binary builder, * invoke default ctor, or allocate). diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 1729a72f70558..cb28b817aa6ca 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -30,12 +30,14 @@ import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import javax.cache.CacheException; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cache.query.QueryCancelledException; +import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.events.CacheQueryExecutedEvent; import org.apache.ignite.events.DiscoveryEvent; @@ -89,6 +91,7 @@ import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.distributedJoinMode; import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.MAP; import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REPLICATED; +import static org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER; import static org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2ValueMessageFactory.toMessages; /** @@ -446,7 +449,7 @@ private void onQueryRequest(final ClusterNode node, final GridH2QueryRequest req req.isFlagSet(GridH2QueryRequest.FLAG_IS_LOCAL), req.isFlagSet(GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS)); - final boolean enforceJoinOrder = req.isFlagSet(GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER); + final boolean enforceJoinOrder = req.isFlagSet(FLAG_ENFORCE_JOIN_ORDER); final boolean explain = req.isFlagSet(GridH2QueryRequest.FLAG_EXPLAIN); final boolean replicated = req.isFlagSet(GridH2QueryRequest.FLAG_REPLICATED); final boolean lazy = req.isFlagSet(GridH2QueryRequest.FLAG_LAZY); @@ -770,14 +773,20 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th MapNodeResults nodeResults = resultsForNode(node.id()); try { - String query = req.queries().get(0).query(); - IndexingQueryFilter filter = h2.backupFilter(req.topologyVersion(), req.queryPartitions()); GridQueryCancel cancel = nodeResults.putUpdate(reqId); - UpdateResult updRes = h2.mapDistributedUpdate(req.mode(), req.schemaName(), req.targetTable(), - req.columnNames(), query, req.parameters(), req.pageSize(), req.timeout(), filter, cancel); + SqlFieldsQuery fldsQry = new SqlFieldsQuery(req.query()); + + if (req.parameters() != null) + fldsQry.setArgs(req.parameters()); + + fldsQry.setEnforceJoinOrder(req.isFlagSet(FLAG_ENFORCE_JOIN_ORDER)); + fldsQry.setTimeout(req.timeout(), TimeUnit.MILLISECONDS); + fldsQry.setPageSize(req.pageSize()); + + UpdateResult updRes = h2.mapDistributedUpdate(req.schemaName(), fldsQry, filter, cancel); sendUpdateResponse(node, reqId, updRes, null); } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index b6d1bb228ac8e..098143f5b48e3 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -847,14 +847,11 @@ public Iterator> query( * * @param schemaName Schema name. * @param cacheIds Cache ids. - * @param updateMode Update mode. - * @param tgtTable Target table. - * @param colNames Column names. * @param selectQry Select query. + * @param params SQL parameters. * @param enforceJoinOrder Enforce join order of tables. * @param pageSize Page size. * @param timeoutMillis Timeout. - * @param params SQL parameters. * @param parts Partitions. * @param cancel Cancel state. * @return Update result. @@ -862,14 +859,11 @@ public Iterator> query( public UpdateResult update( String schemaName, List cacheIds, - byte updateMode, - String tgtTable, - String[] colNames, String selectQry, + Object[] params, boolean enforceJoinOrder, int pageSize, int timeoutMillis, - Object[] params, final int[] parts, GridQueryCancel cancel ) { @@ -889,11 +883,8 @@ public UpdateResult update( GridH2DmlRequest req = new GridH2DmlRequest() .requestId(reqId) .topologyVersion(h2.readyTopologyVersion()) - .mode(updateMode) .schemaName(schemaName) - .targetTable(tgtTable) - .columnNames(colNames) - .queries(Collections.singletonList(new GridCacheSqlQuery(selectQry))) + .query(selectQry) .pageSize(pageSize) .parameters(params) .timeout(timeoutMillis) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java index 91fb6f9288160..27e38cb9ad7e8 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java @@ -18,7 +18,6 @@ import java.io.Externalizable; import java.nio.ByteBuffer; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.UUID; @@ -31,8 +30,6 @@ import org.apache.ignite.internal.binary.BinaryMarshaller; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable; -import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; -import org.apache.ignite.internal.processors.cache.query.QueryTable; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; @@ -74,17 +71,11 @@ public class GridH2DmlRequest implements Message, GridCacheQueryMarshallable { /** Query. */ @GridToStringInclude - @GridDirectCollection(Message.class) - private List qrys; + private String qry; /** */ private byte flags; - /** */ - @GridToStringInclude - @GridDirectCollection(Message.class) - private Collection tbls; - /** */ private int timeout; @@ -99,15 +90,6 @@ public class GridH2DmlRequest implements Message, GridCacheQueryMarshallable { /** Schema name. */ private String schemaName; - /** Update mode. */ - private byte mode; - - /** Target table. */ - private String tgtTbl; //TODO: go for QueryTable instead? - - /** Target table column names. */ - private String[] colNames; - /** * Required by {@link Externalizable} */ @@ -125,16 +107,12 @@ public GridH2DmlRequest(GridH2DmlRequest req) { parts = req.parts; qryParts = req.qryParts; pageSize = req.pageSize; - qrys = req.qrys; + qry = req.qry; flags = req.flags; - tbls = req.tbls; timeout = req.timeout; params = req.params; paramsBytes = req.paramsBytes; schemaName = req.schemaName; - mode = req.mode; - tgtTbl = req.tgtTbl; - colNames = req.colNames; } /** @@ -157,23 +135,6 @@ public GridH2DmlRequest parameters(Object[] params) { return this; } - /** - * @param tbls Tables. - * @return {@code this}. - */ - public GridH2DmlRequest tables(Collection tbls) { - this.tbls = tbls; - - return this; - } - - /** - * @return Tables. - */ - public Collection tables() { - return tbls; - } - /** * @param reqId Request ID. * @return {@code this}. @@ -277,20 +238,20 @@ public int pageSize() { } /** - * @param qrys SQL Queries. + * @param qry SQL Query. * @return {@code this}. */ - public GridH2DmlRequest queries(List qrys) { - this.qrys = qrys; + public GridH2DmlRequest query(String qry) { + this.qry = qry; return this; } /** - * @return SQL Queries. + * @return SQL Query. */ - public List queries() { - return qrys; + public String query() { + return qry; } /** @@ -347,57 +308,6 @@ public GridH2DmlRequest schemaName(String schemaName) { return this; } - /** - * @return Update mode. - */ - public byte mode() { - return mode; - } - - /** - * @param mode Update mode. - * @return {@code this} - */ - public GridH2DmlRequest mode(byte mode) { - this.mode = mode; - - return this; - } - - /** - * @return Target table. - */ - public String targetTable() { - return tgtTbl; - } - - /** - * @param targetTable Target table. - * @return {@code this} - */ - public GridH2DmlRequest targetTable(String targetTable) { - tgtTbl = targetTable; - - return this; - } - - /** - * @return Target table column names. - */ - public String[] columnNames() { - return colNames; - } - - /** - * @param columnNames Target table column names. - * @return {@code this} - */ - public GridH2DmlRequest columnNames(String[] columnNames) { - this.colNames = columnNames; - - return this; - } - /** {@inheritDoc} */ @Override public void marshall(Marshaller m) { if (paramsBytes != null) @@ -454,84 +364,60 @@ public GridH2DmlRequest columnNames(String[] columnNames) { writer.incrementState(); case 1: - if (!writer.writeObjectArray("colNames", colNames, MessageCollectionItemType.STRING)) + if (!writer.writeByte("flags", flags)) return false; writer.incrementState(); case 2: - if (!writer.writeByte("flags", flags)) + if (!writer.writeInt("pageSize", pageSize)) return false; writer.incrementState(); case 3: - if (!writer.writeByte("mode", mode)) + if (!writer.writeByteArray("paramsBytes", paramsBytes)) return false; writer.incrementState(); case 4: - if (!writer.writeInt("pageSize", pageSize)) + if (!writer.writeMap("parts", parts, MessageCollectionItemType.UUID, MessageCollectionItemType.INT_ARR)) return false; writer.incrementState(); case 5: - if (!writer.writeByteArray("paramsBytes", paramsBytes)) + if (!writer.writeString("qry", qry)) return false; writer.incrementState(); case 6: - if (!writer.writeMap("parts", parts, MessageCollectionItemType.UUID, MessageCollectionItemType.INT_ARR)) - return false; - - writer.incrementState(); - - case 7: if (!writer.writeIntArray("qryParts", qryParts)) return false; writer.incrementState(); - case 8: - if (!writer.writeCollection("qrys", qrys, MessageCollectionItemType.MSG)) - return false; - - writer.incrementState(); - - case 9: + case 7: if (!writer.writeLong("reqId", reqId)) return false; writer.incrementState(); - case 10: + case 8: if (!writer.writeString("schemaName", schemaName)) return false; writer.incrementState(); - case 11: - if (!writer.writeCollection("tbls", tbls, MessageCollectionItemType.MSG)) - return false; - - writer.incrementState(); - - case 12: - if (!writer.writeString("tgtTbl", tgtTbl)) - return false; - - writer.incrementState(); - - case 13: + case 9: if (!writer.writeInt("timeout", timeout)) return false; writer.incrementState(); - case 14: + case 10: if (!writer.writeMessage("topVer", topVer)) return false; @@ -559,7 +445,7 @@ public GridH2DmlRequest columnNames(String[] columnNames) { reader.incrementState(); case 1: - colNames = reader.readObjectArray("colNames", MessageCollectionItemType.STRING, String.class); + flags = reader.readByte("flags"); if (!reader.isLastRead()) return false; @@ -567,7 +453,7 @@ public GridH2DmlRequest columnNames(String[] columnNames) { reader.incrementState(); case 2: - flags = reader.readByte("flags"); + pageSize = reader.readInt("pageSize"); if (!reader.isLastRead()) return false; @@ -575,7 +461,7 @@ public GridH2DmlRequest columnNames(String[] columnNames) { reader.incrementState(); case 3: - mode = reader.readByte("mode"); + paramsBytes = reader.readByteArray("paramsBytes"); if (!reader.isLastRead()) return false; @@ -583,7 +469,7 @@ public GridH2DmlRequest columnNames(String[] columnNames) { reader.incrementState(); case 4: - pageSize = reader.readInt("pageSize"); + parts = reader.readMap("parts", MessageCollectionItemType.UUID, MessageCollectionItemType.INT_ARR, false); if (!reader.isLastRead()) return false; @@ -591,7 +477,7 @@ public GridH2DmlRequest columnNames(String[] columnNames) { reader.incrementState(); case 5: - paramsBytes = reader.readByteArray("paramsBytes"); + qry = reader.readString("qry"); if (!reader.isLastRead()) return false; @@ -599,14 +485,6 @@ public GridH2DmlRequest columnNames(String[] columnNames) { reader.incrementState(); case 6: - parts = reader.readMap("parts", MessageCollectionItemType.UUID, MessageCollectionItemType.INT_ARR, false); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); - - case 7: qryParts = reader.readIntArray("qryParts"); if (!reader.isLastRead()) @@ -614,15 +492,7 @@ public GridH2DmlRequest columnNames(String[] columnNames) { reader.incrementState(); - case 8: - qrys = reader.readCollection("qrys", MessageCollectionItemType.MSG); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); - - case 9: + case 7: reqId = reader.readLong("reqId"); if (!reader.isLastRead()) @@ -630,7 +500,7 @@ public GridH2DmlRequest columnNames(String[] columnNames) { reader.incrementState(); - case 10: + case 8: schemaName = reader.readString("schemaName"); if (!reader.isLastRead()) @@ -638,23 +508,7 @@ public GridH2DmlRequest columnNames(String[] columnNames) { reader.incrementState(); - case 11: - tbls = reader.readCollection("tbls", MessageCollectionItemType.MSG); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); - - case 12: - tgtTbl = reader.readString("tgtTbl"); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); - - case 13: + case 9: timeout = reader.readInt("timeout"); if (!reader.isLastRead()) @@ -662,7 +516,7 @@ public GridH2DmlRequest columnNames(String[] columnNames) { reader.incrementState(); - case 14: + case 10: topVer = reader.readMessage("topVer"); if (!reader.isLastRead()) @@ -682,7 +536,7 @@ public GridH2DmlRequest columnNames(String[] columnNames) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 15; + return 11; } /** {@inheritDoc} */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java index 3e218a892382c..e970d5f3ddbdd 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java @@ -18,14 +18,23 @@ package org.apache.ignite.internal.processors.query.h2.twostep.msg; import java.nio.ByteBuffer; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.GridDirectTransient; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.binary.BinaryMarshaller; +import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable; +import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.marshaller.Marshaller; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType; import org.apache.ignite.plugin.extensions.communication.MessageReader; import org.apache.ignite.plugin.extensions.communication.MessageWriter; /** Response to remote DML request. */ -public class GridH2DmlResponse implements Message { +public class GridH2DmlResponse implements Message, GridCacheQueryMarshallable { /** */ private static final long serialVersionUID = 0L; @@ -51,8 +60,13 @@ public class GridH2DmlResponse implements Message { private String err; /** Keys that failed. */ + @GridToStringInclude + @GridDirectTransient private Object[] errKeys; + /** */ + private byte[] errKeysBytes; + /** * Default constructor. */ @@ -112,6 +126,39 @@ public String error() { return err; } + /** {@inheritDoc} */ + @Override public void marshall(Marshaller m) { + if (errKeysBytes != null || errKeys == null) + return; + + try { + errKeysBytes = U.marshal(m, errKeys); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + + /** {@inheritDoc} */ + @SuppressWarnings("IfMayBeConditional") + @Override public void unmarshall(Marshaller m, GridKernalContext ctx) { + if (errKeys != null || errKeysBytes == null) + return; + + try { + final ClassLoader ldr = U.resolveClassLoader(ctx.config()); + + if (m instanceof BinaryMarshaller) + // To avoid deserializing of enum types. + errKeys = ((BinaryMarshaller)m).binaryMarshaller().unmarshal(errKeysBytes, ldr); + else + errKeys = U.unmarshal(m, errKeysBytes, ldr); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(GridH2DmlResponse.class, this); @@ -136,7 +183,7 @@ public String error() { writer.incrementState(); case 1: - if (!writer.writeObjectArray("errKeys", errKeys, MessageCollectionItemType.MSG)) + if (!writer.writeByteArray("errKeysBytes", errKeysBytes)) return false; writer.incrementState(); @@ -181,7 +228,7 @@ public String error() { reader.incrementState(); case 1: - errKeys = reader.readObjectArray("errKeys", MessageCollectionItemType.MSG, Object.class); + errKeysBytes = reader.readByteArray("errKeysBytes"); if (!reader.isLastRead()) return false; From a2168582e85299f53d7f1d5ebabb2724f50a8138 Mon Sep 17 00:00:00 2001 From: devozerov Date: Wed, 23 Aug 2017 16:58:00 +0300 Subject: [PATCH 08/40] Review comments. --- .../internal/processors/query/h2/DmlStatementsProcessor.java | 2 ++ .../ignite/internal/processors/query/h2/dml/UpdatePlan.java | 1 + .../processors/query/h2/twostep/DistributedUpdateRun.java | 4 +++- .../processors/query/h2/twostep/GridMapQueryExecutor.java | 1 + .../processors/query/h2/twostep/GridReduceQueryExecutor.java | 4 ++++ 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 8aa00fd79278a..42a02eb87a4c6 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -237,6 +237,8 @@ private void checkPlanCanBeDistributed(SqlFieldsQuery fieldsQry, PreparedStateme fieldsQry.isDistributedJoins(), fieldsQry.isEnforceJoinOrder(), idx); + // TODO: replicatedOnly -> route ro single server + plan.distributed = !qry.isReplicatedOnly() && qry.skipMergeTable() && qry.mapQueries().size() == 1 && diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index a0fa409380290..1978137bde6c0 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -134,6 +134,7 @@ public static UpdatePlan forFastUpdate(UpdateMode mode, GridH2Table tbl, FastUpd } /** */ + // TODO: Remove public static UpdatePlan fromMessage(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] colTypes, String qry, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx) { return new UpdatePlan(mode, tbl, colNames, colTypes, keySupplier, valSupplier, keyColIdx, valColIdx, qry, false, diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java index 25c946b34b544..73e948f58b7f2 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java @@ -35,6 +35,7 @@ class DistributedUpdateRun { private AtomicLong rspCntr; /** Update counter. */ + // TODO: Track concrete nodes instead of plain counter. private AtomicLong updCntr = new AtomicLong(); /** Error keys. */ @@ -77,6 +78,7 @@ GridFutureAdapter future() { * @param e Pre-formatted error. */ void handleDisconnect(CacheException e) { + // TODO: Snesible error message. fut.onDone(new IgniteCheckedException("Update failed.", e)); } @@ -86,7 +88,7 @@ void handleDisconnect(CacheException e) { * @param nodeId Node id. */ void handleNodeLeft(UUID nodeId) { - fut.onDone(new IgniteCheckedException("Update failed: node " + nodeId + " has left")); + fut.onDone(new IgniteCheckedException("Update failed because map node left topology [nodeId=" + nodeId + "]")); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index cb28b817aa6ca..ec482a6226999 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -765,6 +765,7 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th if (!reservePartitions(cacheIds, topVer, parts, reserved)) { log.error("Failed to reserve partitions for DML request"); + // TODO: Explanation (Suggestion), sendUpdateResponse(node, reqId, null, "Failed to reserve partitions for DML request"); return; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index 098143f5b48e3..27f27217b5f75 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -106,6 +106,8 @@ import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REDUCE; import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter.mergeTableIdentifier; +// TODO: Logging + /** * Reduce query executor. */ @@ -135,6 +137,7 @@ public class GridReduceQueryExecutor { private final ConcurrentMap runs = new ConcurrentHashMap8<>(); /** Contexts of running DML requests. */ + // TODO: Use normal ConcurrentHashMap instead. private final ConcurrentMap updRuns = new ConcurrentHashMap8<>(); /** */ @@ -941,6 +944,7 @@ private void onDmlResponse(final ClusterNode node, GridH2DmlResponse msg) { DistributedUpdateRun r = updRuns.get(reqId); if (r == null) + // TODO: Print warning return; if (r.handleResponse(node.id(), msg)) From 2f26e387c3d966ca17bcccebe8f18363c1ca48f8 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Wed, 23 Aug 2017 18:25:50 +0300 Subject: [PATCH 09/40] IGNITE-6024: fixing review comments --- .../ignite/codegen/MessageCodeGenerator.java | 8 +-- .../processors/query/h2/dml/UpdatePlan.java | 8 --- .../h2/twostep/DistributedUpdateRun.java | 61 ++++++++-------- .../h2/twostep/GridMapQueryExecutor.java | 16 ++--- .../h2/twostep/GridReduceQueryExecutor.java | 5 +- .../h2/twostep/msg/GridH2DmlRequest.java | 70 ++++--------------- .../h2/twostep/msg/GridH2DmlResponse.java | 44 ++---------- 7 files changed, 58 insertions(+), 154 deletions(-) diff --git a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java index b5499b47a3812..3ea0c81ab7295 100644 --- a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java +++ b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java @@ -46,8 +46,6 @@ import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx; -import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlRequest; -import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlResponse; import org.apache.ignite.internal.util.typedef.internal.SB; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteUuid; @@ -163,7 +161,7 @@ private static MessageCollectionItemType typeEnum(Class cls) { * @throws Exception In case of error. */ public static void main(String[] args) throws Exception { - String srcDir = INDEXING_SRC_DIR;//DFLT_SRC_DIR; + String srcDir = DFLT_SRC_DIR; if (args != null && args.length > 0) srcDir = args[0]; @@ -237,8 +235,8 @@ public static void main(String[] args) throws Exception { // gen.generateAndWrite(GridH2RowMessage.class); // gen.generateAndWrite(GridCacheVersion.class); // gen.generateAndWrite(GridCacheVersionEx.class); - gen.generateAndWrite(GridH2DmlRequest.class); - gen.generateAndWrite(GridH2DmlResponse.class); +// gen.generateAndWrite(GridH2DmlRequest.class); +// gen.generateAndWrite(GridH2DmlResponse.class); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 1978137bde6c0..745095d4f1e89 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -132,12 +132,4 @@ public static UpdatePlan forFastUpdate(UpdateMode mode, GridH2Table tbl, FastUpd return new UpdatePlan(mode, tbl, null, null, null, null, -1, -1, null, false, 0, fastUpdateArgs); } - - /** */ - // TODO: Remove - public static UpdatePlan fromMessage(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] colTypes, - String qry, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx) { - return new UpdatePlan(mode, tbl, colNames, colTypes, keySupplier, valSupplier, keyColIdx, valColIdx, qry, false, - 0, null); - } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java index 73e948f58b7f2..a2d86a4c49ab1 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java @@ -31,21 +31,23 @@ /** Context for DML operation on reducer node. */ class DistributedUpdateRun { - /** Response counter. */ - private AtomicLong rspCntr; + /** Expected number of responses. */ + private final int nodeCount; - /** Update counter. */ - // TODO: Track concrete nodes instead of plain counter. - private AtomicLong updCntr = new AtomicLong(); + /** Registers nodes that have responded. */ + private final GridConcurrentHashSet rspNodes; - /** Error keys. */ - private GridConcurrentHashSet errorKeys = new GridConcurrentHashSet<>(); + /** Accumulates total number of updated rows. */ + private final AtomicLong updCntr = new AtomicLong(); + + /** Accumulates error keys. */ + private final GridConcurrentHashSet errorKeys = new GridConcurrentHashSet<>(); /** Query info. */ private final GridRunningQueryInfo qry; /** Result future. */ - private GridFutureAdapter fut = new GridFutureAdapter<>(); + private final GridFutureAdapter fut = new GridFutureAdapter<>(); /** * Constructor. @@ -54,9 +56,10 @@ class DistributedUpdateRun { * @param qry Query info. */ DistributedUpdateRun(int nodeCount, GridRunningQueryInfo qry) { - rspCntr = new AtomicLong(nodeCount); - + this.nodeCount = nodeCount; this.qry = qry; + + rspNodes = new GridConcurrentHashSet<>(nodeCount); } /** @@ -99,35 +102,29 @@ void handleNodeLeft(UUID nodeId) { * @return {@code true} if no more responses are expected. */ boolean handleResponse(UUID id, GridH2DmlResponse msg) { - boolean done = rspCntr.decrementAndGet() == 0; - - switch (msg.status()) { - case GridH2DmlResponse.STATUS_ERR_KEYS: - if (!F.isEmpty(msg.errorKeys())) - errorKeys.addAll(Arrays.asList(msg.errorKeys())); + if (!rspNodes.add(id)) + return false; // ignore duplicated messages - // Intentional fall-through. - case GridH2DmlResponse.STATUS_OK: - updCntr.addAndGet(msg.updateCounter()); + String err = msg.error(); - if (done) - fut.onDone(new UpdateResult(updCntr.get(), errorKeys.toArray())); + if (err != null) { + fut.onDone(new IgniteCheckedException("Update failed. " + (F.isEmpty(err)? "" : err) + "[reqId=" + + msg.requestId() + ", node=" + id + "].")); - return done; - - case GridH2DmlResponse.STATUS_ERROR: - String err = msg.error(); + return true; + } - fut.onDone(new IgniteCheckedException("Update failed. " + (F.isEmpty(err)? "" : err) + "[reqId=" + - msg.requestId() + ", node=" + id + "].")); + if (!F.isEmpty(msg.errorKeys())) + errorKeys.addAll(Arrays.asList(msg.errorKeys())); - return false; + long cntr = updCntr.addAndGet(msg.updateCounter()); - default: - fut.onDone(new IgniteCheckedException("Update failed. Invalid status in response message [reqId=" + - msg.requestId() + ", node=" + id + "].")); + if (rspNodes.size() == nodeCount) { + fut.onDone(new UpdateResult(cntr, errorKeys.toArray())); - return false; + return true; } + + return false; } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index ec482a6226999..8f0ba256a4e3a 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -748,13 +748,9 @@ private void onQueryRequest0( * @param req DML request. */ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) throws IgniteCheckedException { - int[] qryParts = req.queryPartitions(); - - final Map partsMap = req.partitions(); + int[] parts = req.queryPartitions(); - final int[] parts = qryParts == null ? partsMap == null ? null : partsMap.get(ctx.localNodeId()) : qryParts; - - final List cacheIds = req.caches(); + List cacheIds = req.caches(); long reqId = req.requestId(); @@ -766,6 +762,7 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th log.error("Failed to reserve partitions for DML request"); // TODO: Explanation (Suggestion), + // Explanation (Retry your request when re-balancing is over) sendUpdateResponse(node, reqId, null, "Failed to reserve partitions for DML request"); return; @@ -774,7 +771,7 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th MapNodeResults nodeResults = resultsForNode(node.id()); try { - IndexingQueryFilter filter = h2.backupFilter(req.topologyVersion(), req.queryPartitions()); + IndexingQueryFilter filter = h2.backupFilter(req.topologyVersion(), parts); GridQueryCancel cancel = nodeResults.putUpdate(reqId); @@ -841,10 +838,7 @@ private void sendError(ClusterNode node, long qryReqId, Throwable err) { */ private void sendUpdateResponse(ClusterNode node, long reqId, UpdateResult updResult, String error) { try { - byte status = (error != null || updResult == null) ? GridH2DmlResponse.STATUS_ERROR : - F.isEmpty(updResult.errorKeys()) ? GridH2DmlResponse.STATUS_OK : GridH2DmlResponse.STATUS_ERR_KEYS; - - GridH2DmlResponse rsp = new GridH2DmlResponse(reqId, status, updResult == null ? 0 : updResult.counter(), + GridH2DmlResponse rsp = new GridH2DmlResponse(reqId, updResult == null ? 0 : updResult.counter(), updResult == null ? null : updResult.errorKeys(), error); if (node.isLocal()) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index 27f27217b5f75..83f213593d2f1 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -137,8 +138,7 @@ public class GridReduceQueryExecutor { private final ConcurrentMap runs = new ConcurrentHashMap8<>(); /** Contexts of running DML requests. */ - // TODO: Use normal ConcurrentHashMap instead. - private final ConcurrentMap updRuns = new ConcurrentHashMap8<>(); + private final ConcurrentMap updRuns = new ConcurrentHashMap<>(); /** */ private volatile List fakeTbls = Collections.emptyList(); @@ -891,7 +891,6 @@ public UpdateResult update( .pageSize(pageSize) .parameters(params) .timeout(timeoutMillis) - .partitions(convert(nodesParts.partitionsMap())) .flags(enforceJoinOrder ? GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER : 0); updRuns.put(reqId, r); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java index 27e38cb9ad7e8..2ceadb51b7dff 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java @@ -19,12 +19,9 @@ import java.io.Externalizable; import java.nio.ByteBuffer; import java.util.List; -import java.util.Map; -import java.util.UUID; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.internal.GridDirectCollection; -import org.apache.ignite.internal.GridDirectMap; import org.apache.ignite.internal.GridDirectTransient; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.binary.BinaryMarshaller; @@ -57,11 +54,6 @@ public class GridH2DmlRequest implements Message, GridCacheQueryMarshallable { /** Topology version. */ private AffinityTopologyVersion topVer; - /** Explicit partitions mappings for nodes. */ - @GridToStringInclude - @GridDirectMap(keyType = UUID.class, valueType = int[].class) - private Map parts; - /** Query partitions. */ @GridToStringInclude private int[] qryParts; @@ -73,18 +65,18 @@ public class GridH2DmlRequest implements Message, GridCacheQueryMarshallable { @GridToStringInclude private String qry; - /** */ + /** Flags. */ private byte flags; - /** */ + /** Timeout. */ private int timeout; - /** */ + /** Query parameters. */ @GridToStringInclude(sensitive = true) @GridDirectTransient private Object[] params; - /** */ + /** Query parameters as bytes. */ private byte[] paramsBytes; /** Schema name. */ @@ -104,7 +96,6 @@ public GridH2DmlRequest(GridH2DmlRequest req) { reqId = req.reqId; caches = req.caches; topVer = req.topVer; - parts = req.parts; qryParts = req.qryParts; pageSize = req.pageSize; qry = req.qry; @@ -186,23 +177,6 @@ public AffinityTopologyVersion topologyVersion() { return topVer; } - /** - * @return Explicit partitions mapping. - */ - public Map partitions() { - return parts; - } - - /** - * @param parts Explicit partitions mapping. - * @return {@code this}. - */ - public GridH2DmlRequest partitions(Map parts) { - this.parts = parts; - - return this; - } - /** * @return Query partitions. */ @@ -382,42 +356,36 @@ public GridH2DmlRequest schemaName(String schemaName) { writer.incrementState(); case 4: - if (!writer.writeMap("parts", parts, MessageCollectionItemType.UUID, MessageCollectionItemType.INT_ARR)) - return false; - - writer.incrementState(); - - case 5: if (!writer.writeString("qry", qry)) return false; writer.incrementState(); - case 6: + case 5: if (!writer.writeIntArray("qryParts", qryParts)) return false; writer.incrementState(); - case 7: + case 6: if (!writer.writeLong("reqId", reqId)) return false; writer.incrementState(); - case 8: + case 7: if (!writer.writeString("schemaName", schemaName)) return false; writer.incrementState(); - case 9: + case 8: if (!writer.writeInt("timeout", timeout)) return false; writer.incrementState(); - case 10: + case 9: if (!writer.writeMessage("topVer", topVer)) return false; @@ -469,14 +437,6 @@ public GridH2DmlRequest schemaName(String schemaName) { reader.incrementState(); case 4: - parts = reader.readMap("parts", MessageCollectionItemType.UUID, MessageCollectionItemType.INT_ARR, false); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); - - case 5: qry = reader.readString("qry"); if (!reader.isLastRead()) @@ -484,7 +444,7 @@ public GridH2DmlRequest schemaName(String schemaName) { reader.incrementState(); - case 6: + case 5: qryParts = reader.readIntArray("qryParts"); if (!reader.isLastRead()) @@ -492,7 +452,7 @@ public GridH2DmlRequest schemaName(String schemaName) { reader.incrementState(); - case 7: + case 6: reqId = reader.readLong("reqId"); if (!reader.isLastRead()) @@ -500,7 +460,7 @@ public GridH2DmlRequest schemaName(String schemaName) { reader.incrementState(); - case 8: + case 7: schemaName = reader.readString("schemaName"); if (!reader.isLastRead()) @@ -508,7 +468,7 @@ public GridH2DmlRequest schemaName(String schemaName) { reader.incrementState(); - case 9: + case 8: timeout = reader.readInt("timeout"); if (!reader.isLastRead()) @@ -516,7 +476,7 @@ public GridH2DmlRequest schemaName(String schemaName) { reader.incrementState(); - case 10: + case 9: topVer = reader.readMessage("topVer"); if (!reader.isLastRead()) @@ -536,7 +496,7 @@ public GridH2DmlRequest schemaName(String schemaName) { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 11; + return 10; } /** {@inheritDoc} */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java index e970d5f3ddbdd..513f2f85e5e99 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java @@ -29,7 +29,6 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.marshaller.Marshaller; import org.apache.ignite.plugin.extensions.communication.Message; -import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType; import org.apache.ignite.plugin.extensions.communication.MessageReader; import org.apache.ignite.plugin.extensions.communication.MessageWriter; @@ -38,22 +37,10 @@ public class GridH2DmlResponse implements Message, GridCacheQueryMarshallable { /** */ private static final long serialVersionUID = 0L; - /** Successful execution. */ - public static final byte STATUS_OK = 0; - - /** Error occurred. */ - public static final byte STATUS_ERROR = 1; - - /** Partial success. Erroneous keys identified. */ - public static final byte STATUS_ERR_KEYS = 2; - /** Request id. */ private long reqId; - /** Status. */ - private byte status; - - /** Updated rows counter. */ + /** Number of updated rows. */ private long updCnt; /** Error message. */ @@ -64,7 +51,7 @@ public class GridH2DmlResponse implements Message, GridCacheQueryMarshallable { @GridDirectTransient private Object[] errKeys; - /** */ + /** Keys that failed (after marshalling). */ private byte[] errKeysBytes; /** @@ -78,14 +65,12 @@ public GridH2DmlResponse() { * Constructor. * * @param reqId Request id. - * @param status Status. * @param updCnt Updated row number. * @param errKeys Erroneous keys. * @param error Error message. */ - public GridH2DmlResponse(long reqId, byte status, long updCnt, Object[] errKeys, String error) { + public GridH2DmlResponse(long reqId, long updCnt, Object[] errKeys, String error) { this.reqId = reqId; - this.status = status; this.updCnt = updCnt; this.errKeys = errKeys; this.err = error; @@ -98,13 +83,6 @@ public long requestId() { return reqId; } - /** - * @return Status. - */ - public byte status() { - return status; - } - /** * @return Update counter. */ @@ -195,12 +173,6 @@ public String error() { writer.incrementState(); case 3: - if (!writer.writeByte("status", status)) - return false; - - writer.incrementState(); - - case 4: if (!writer.writeLong("updCnt", updCnt)) return false; @@ -244,14 +216,6 @@ public String error() { reader.incrementState(); case 3: - status = reader.readByte("status"); - - if (!reader.isLastRead()) - return false; - - reader.incrementState(); - - case 4: updCnt = reader.readLong("updCnt"); if (!reader.isLastRead()) @@ -271,7 +235,7 @@ public String error() { /** {@inheritDoc} */ @Override public byte fieldsCount() { - return 5; + return 4; } @Override public void onAckReceived() { From 896418805c7337e7cf2ad3c17293a337bd6d788e Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Wed, 23 Aug 2017 20:22:37 +0300 Subject: [PATCH 10/40] IGNITE-6024: fixed review comments --- .../query/h2/DmlStatementsProcessor.java | 11 ++-- .../processors/query/h2/IgniteH2Indexing.java | 4 +- .../processors/query/h2/dml/UpdatePlan.java | 3 ++ .../h2/twostep/DistributedUpdateRun.java | 17 ++---- .../h2/twostep/GridMapQueryExecutor.java | 25 +++++---- .../h2/twostep/GridReduceQueryExecutor.java | 53 +++++++++++++------ .../h2/twostep/msg/GridH2DmlRequest.java | 3 ++ .../h2/twostep/msg/GridH2DmlResponse.java | 3 ++ 8 files changed, 75 insertions(+), 44 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 42a02eb87a4c6..7147db5b88ef9 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -237,15 +237,14 @@ private void checkPlanCanBeDistributed(SqlFieldsQuery fieldsQry, PreparedStateme fieldsQry.isDistributedJoins(), fieldsQry.isEnforceJoinOrder(), idx); - // TODO: replicatedOnly -> route ro single server - - plan.distributed = !qry.isReplicatedOnly() && - qry.skipMergeTable() && + plan.distributed = qry.skipMergeTable() && qry.mapQueries().size() == 1 && !qry.mapQueries().get(0).hasSubQueries(); - if (plan.distributed) + if (plan.distributed) { plan.cacheIds = idx.collectCacheIds(CU.cacheId(plan.tbl.cacheName()), qry); + plan.isReplicatedOnly = qry.isReplicatedOnly(); + } } } catch (SQLException e) { @@ -554,7 +553,7 @@ private UpdateResult doDistributedUpdate(String schemaName, SqlFieldsQuery field if (cancel == null) cancel = new GridQueryCancel(); - return idx.runDistributedUpdate(schemaName, fieldsQry, plan.cacheIds, cancel); + return idx.runDistributedUpdate(schemaName, fieldsQry, plan.cacheIds, plan.isReplicatedOnly, cancel); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 0273b6735c5bb..446f17dc79243 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1198,6 +1198,7 @@ private Iterable> runQueryTwoStep( * @param schemaName Schema name. * @param fieldsQry Initial update query. * @param cacheIds Cache identifiers. + * @param isReplicatedOnly Whether query uses only replicated caches. * @param cancel Cancel state. * @return Update result. */ @@ -1205,10 +1206,11 @@ UpdateResult runDistributedUpdate( String schemaName, SqlFieldsQuery fieldsQry, List cacheIds, + boolean isReplicatedOnly, GridQueryCancel cancel) { return rdcQryExec.update(schemaName, cacheIds, fieldsQry.getSql(), fieldsQry.getArgs(), fieldsQry.isEnforceJoinOrder(), fieldsQry.getPageSize(), fieldsQry.getTimeout(), - fieldsQry.getPartitions(), cancel); + fieldsQry.getPartitions(), isReplicatedOnly, cancel); } /** {@inheritDoc} */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 745095d4f1e89..028df96e333bb 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -71,6 +71,9 @@ public final class UpdatePlan { /** Integer cache identifiers. */ public List cacheIds; + /** Flag that query contains only replicated tables. */ + public boolean isReplicatedOnly; + /** */ private UpdatePlan(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry, diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java index a2d86a4c49ab1..2a26aa775fd58 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java @@ -81,8 +81,7 @@ GridFutureAdapter future() { * @param e Pre-formatted error. */ void handleDisconnect(CacheException e) { - // TODO: Snesible error message. - fut.onDone(new IgniteCheckedException("Update failed.", e)); + fut.onDone(new IgniteCheckedException("Update failed because client node have disconnected.", e)); } /** @@ -99,11 +98,10 @@ void handleNodeLeft(UUID nodeId) { * * @param id Node id. * @param msg Response message. - * @return {@code true} if no more responses are expected. */ - boolean handleResponse(UUID id, GridH2DmlResponse msg) { + void handleResponse(UUID id, GridH2DmlResponse msg) { if (!rspNodes.add(id)) - return false; // ignore duplicated messages + return; // ignore duplicated messages String err = msg.error(); @@ -111,7 +109,7 @@ boolean handleResponse(UUID id, GridH2DmlResponse msg) { fut.onDone(new IgniteCheckedException("Update failed. " + (F.isEmpty(err)? "" : err) + "[reqId=" + msg.requestId() + ", node=" + id + "].")); - return true; + return; } if (!F.isEmpty(msg.errorKeys())) @@ -119,12 +117,7 @@ boolean handleResponse(UUID id, GridH2DmlResponse msg) { long cntr = updCntr.addAndGet(msg.updateCounter()); - if (rspNodes.size() == nodeCount) { + if (rspNodes.size() == nodeCount) fut.onDone(new UpdateResult(cntr, errorKeys.toArray())); - - return true; - } - - return false; } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 8f0ba256a4e3a..3fc8904c3dac1 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -21,6 +21,7 @@ import java.sql.ResultSet; import java.util.AbstractCollection; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -91,7 +92,6 @@ import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.distributedJoinMode; import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.MAP; import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REPLICATED; -import static org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER; import static org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2ValueMessageFactory.toMessages; /** @@ -449,7 +449,7 @@ private void onQueryRequest(final ClusterNode node, final GridH2QueryRequest req req.isFlagSet(GridH2QueryRequest.FLAG_IS_LOCAL), req.isFlagSet(GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS)); - final boolean enforceJoinOrder = req.isFlagSet(FLAG_ENFORCE_JOIN_ORDER); + final boolean enforceJoinOrder = req.isFlagSet(GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER); final boolean explain = req.isFlagSet(GridH2QueryRequest.FLAG_EXPLAIN); final boolean replicated = req.isFlagSet(GridH2QueryRequest.FLAG_REPLICATED); final boolean lazy = req.isFlagSet(GridH2QueryRequest.FLAG_LAZY); @@ -759,11 +759,12 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th List reserved = new ArrayList<>(); if (!reservePartitions(cacheIds, topVer, parts, reserved)) { - log.error("Failed to reserve partitions for DML request"); + U.error(log, "Failed to reserve partitions for DML request. [localNodeId=" + ctx.localNodeId() + + ", nodeId=" + node.id() + ", reqId=" + req.requestId() + ", cacheIds=" + cacheIds + + ", topVer=" + topVer + ", parts=" + Arrays.toString(parts) + ']'); - // TODO: Explanation (Suggestion), - // Explanation (Retry your request when re-balancing is over) - sendUpdateResponse(node, reqId, null, "Failed to reserve partitions for DML request"); + sendUpdateResponse(node, reqId, null, "Failed to reserve partitions for DML request. " + + "Explanation (Retry your request when re-balancing is over)."); return; } @@ -780,7 +781,7 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th if (req.parameters() != null) fldsQry.setArgs(req.parameters()); - fldsQry.setEnforceJoinOrder(req.isFlagSet(FLAG_ENFORCE_JOIN_ORDER)); + fldsQry.setEnforceJoinOrder(req.isFlagSet(GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER)); fldsQry.setTimeout(req.timeout(), TimeUnit.MILLISECONDS); fldsQry.setPageSize(req.pageSize()); @@ -789,9 +790,8 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th sendUpdateResponse(node, reqId, updRes, null); } catch (Exception e) { - log.error("Error processing dml request " + e.toString()); - - nodeResults.removeUpdate(reqId); + U.error(log, "Error processing dml request. [localNodeId=" + ctx.localNodeId() + + ", nodeId=" + node.id() + ", req=" + req + ']', e); sendUpdateResponse(node, reqId, null, e.getMessage()); } @@ -801,6 +801,8 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th for (int i = 0; i < reserved.size(); i++) reserved.get(i).release(); } + + nodeResults.removeUpdate(reqId); } } @@ -841,6 +843,9 @@ private void sendUpdateResponse(ClusterNode node, long reqId, UpdateResult updRe GridH2DmlResponse rsp = new GridH2DmlResponse(reqId, updResult == null ? 0 : updResult.counter(), updResult == null ? null : updResult.errorKeys(), error); + if (log.isDebugEnabled()) + log.debug("Sending: [localNodeId=" + ctx.localNodeId() + ", node=" + node.id() + ", msg=" + rsp + "]"); + if (node.isLocal()) h2.reduceQueryExecutor().onMessage(ctx.localNodeId(), rsp); else diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index 83f213593d2f1..35004f675d906 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -107,8 +107,6 @@ import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REDUCE; import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter.mergeTableIdentifier; -// TODO: Logging - /** * Reduce query executor. */ @@ -856,6 +854,7 @@ public Iterator> query( * @param pageSize Page size. * @param timeoutMillis Timeout. * @param parts Partitions. + * @param isReplicatedOnly Whether query uses only replicated caches. * @param cancel Cancel state. * @return Update result. */ @@ -868,24 +867,39 @@ public UpdateResult update( int pageSize, int timeoutMillis, final int[] parts, + boolean isReplicatedOnly, GridQueryCancel cancel ) { AffinityTopologyVersion topVer = h2.readyTopologyVersion(); - NodesForPartitionsResult nodesParts = nodesForPartitions(cacheIds, topVer, parts, false); + NodesForPartitionsResult nodesParts = nodesForPartitions(cacheIds, topVer, parts, isReplicatedOnly); final long reqId = qryIdGen.incrementAndGet(); final GridRunningQueryInfo qryInfo = new GridRunningQueryInfo(reqId, selectQry, GridCacheQueryType.SQL_FIELDS, schemaName, U.currentTimeMillis(), cancel, false); - final Collection nodes = nodesParts.nodes(); + Collection nodes = nodesParts.nodes(); + + if (nodes == null) + throw new CacheException("Failed to determine nodes participating in the update. " + + "Explanation (Retry update once topology recovers)."); + + if (isReplicatedOnly) { + ClusterNode locNode = ctx.discovery().localNode(); + + if (nodes.contains(locNode)) + nodes = singletonList(locNode); + else + nodes = singletonList(F.rand(nodes)); + } final DistributedUpdateRun r = new DistributedUpdateRun(nodes.size(), qryInfo); GridH2DmlRequest req = new GridH2DmlRequest() .requestId(reqId) .topologyVersion(h2.readyTopologyVersion()) + .caches(cacheIds) .schemaName(schemaName) .query(selectQry) .pageSize(pageSize) @@ -898,28 +912,34 @@ public UpdateResult update( boolean release = false; try { + Map partsMap = (nodesParts.queryPartitionsMap() != null) ? + nodesParts.queryPartitionsMap() : nodesParts.partitionsMap(); + ExplicitPartitionsSpecializer partsSpec = (parts == null) ? null : - new ExplicitPartitionsSpecializer(nodesParts.queryPartitionsMap()); + new ExplicitPartitionsSpecializer(partsMap); + + final Collection finalNodes = nodes; cancel.set(new Runnable() { @Override public void run() { r.future().onCancelled(); - send(nodes, new GridQueryCancelRequest(reqId), null, false); + send(finalNodes, new GridQueryCancelRequest(reqId), null, false); } }); + // send() logs the debug message if (send(nodes, req, partsSpec, false)) return r.future().get(); - throw new CacheException("Failed to send update request."); + throw new CacheException("Failed to send update request to participating nodes."); } catch (IgniteCheckedException | RuntimeException e) { release = true; - log.error("Error in update: " + e.toString()); + U.error(log, "Error during update [localNodeId=" + ctx.localNodeId() + "]", e); - throw new CacheException("Failed to run update: ", e); + throw new CacheException("Failed to run update. " + e.getMessage(), e); } finally { if (release) @@ -942,15 +962,18 @@ private void onDmlResponse(final ClusterNode node, GridH2DmlResponse msg) { DistributedUpdateRun r = updRuns.get(reqId); - if (r == null) - // TODO: Print warning + if (r == null) { + U.warn(log, "Unexpected dml response (will ignore). [localNodeId=" + ctx.localNodeId() + ", nodeId=" + + node.id() + ", msg=" + msg.toString() + ']'); + return; + } - if (r.handleResponse(node.id(), msg)) - updRuns.remove(reqId); + r.handleResponse(node.id(), msg); } catch (Exception e) { - log.error("Error in dml response processing: " + e.toString()); + U.error(log, "Error in dml response processing. [localNodeId=" + ctx.localNodeId() + ", nodeId=" + + node.id() + ", msg=" + msg.toString() + ']', e); } } @@ -1423,7 +1446,7 @@ private static Map convert(Map m) { * * @param cacheIds Cache ids. * @param topVer Topology version. - * @param parts Paritions array. + * @param parts Partitions array. * @param isReplicatedOnly Allow only replicated caches. * @return Result. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java index 2ceadb51b7dff..a0b96422d37d5 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java @@ -44,6 +44,7 @@ public class GridH2DmlRequest implements Message, GridCacheQueryMarshallable { private static final long serialVersionUID = 0L; /** Request id. */ + @GridToStringInclude private long reqId; /** Cache identifiers. */ @@ -52,6 +53,7 @@ public class GridH2DmlRequest implements Message, GridCacheQueryMarshallable { private List caches; /** Topology version. */ + @GridToStringInclude private AffinityTopologyVersion topVer; /** Query partitions. */ @@ -80,6 +82,7 @@ public class GridH2DmlRequest implements Message, GridCacheQueryMarshallable { private byte[] paramsBytes; /** Schema name. */ + @GridToStringInclude private String schemaName; /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java index 513f2f85e5e99..04502cd403020 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java @@ -38,12 +38,15 @@ public class GridH2DmlResponse implements Message, GridCacheQueryMarshallable { private static final long serialVersionUID = 0L; /** Request id. */ + @GridToStringInclude private long reqId; /** Number of updated rows. */ + @GridToStringInclude private long updCnt; /** Error message. */ + @GridToStringInclude private String err; /** Keys that failed. */ From 791a998d0c3351a3f268d2907704f56eb5bbc926 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Thu, 24 Aug 2017 16:41:45 +0300 Subject: [PATCH 11/40] IGNITE-6024: minors --- .../processors/query/h2/DmlStatementsProcessor.java | 9 ++++++++- .../query/h2/twostep/GridMapQueryExecutor.java | 2 +- .../query/h2/twostep/GridReduceQueryExecutor.java | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 7147db5b88ef9..71203267fe1b2 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -546,7 +546,14 @@ private static UpdateResult doFastUpdate(UpdatePlan plan, Object[] args) throws } } - /** */ + /** + * @param schemaName Schema name. + * @param fieldsQry Initial query. + * @param plan Update plan. + * @param cancel Cancel state. + * @return Update result. + * @throws IgniteCheckedException if failed. + */ private UpdateResult doDistributedUpdate(String schemaName, SqlFieldsQuery fieldsQry, UpdatePlan plan, GridQueryCancel cancel) throws IgniteCheckedException { diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 3fc8904c3dac1..f1e47d3f76d69 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -772,7 +772,7 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th MapNodeResults nodeResults = resultsForNode(node.id()); try { - IndexingQueryFilter filter = h2.backupFilter(req.topologyVersion(), parts); + IndexingQueryFilter filter = h2.backupFilter(topVer, parts); GridQueryCancel cancel = nodeResults.putUpdate(reqId); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index 35004f675d906..9818085feb2ff 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -898,7 +898,7 @@ public UpdateResult update( GridH2DmlRequest req = new GridH2DmlRequest() .requestId(reqId) - .topologyVersion(h2.readyTopologyVersion()) + .topologyVersion(topVer) .caches(cacheIds) .schemaName(schemaName) .query(selectQry) From d4f6771ab6c892ed6741a0cc6d0e186c5a0ede00 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Thu, 31 Aug 2017 14:41:18 +0300 Subject: [PATCH 12/40] IGNITE-6024: added tests --- .../query/h2/DmlStatementsProcessor.java | 21 +- .../processors/query/h2/IgniteH2Indexing.java | 5 +- .../query/h2/sql/GridSqlQuerySplitter.java | 2 + .../h2/twostep/GridMapQueryExecutor.java | 45 +- .../h2/twostep/GridReduceQueryExecutor.java | 7 +- .../IgniteSqlDistributedDmlSelfTest.java | 630 ++++++++++++++++++ .../IgniteCacheQuerySelfTestSuite.java | 2 + 7 files changed, 704 insertions(+), 8 deletions(-) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 71203267fe1b2..05a146ab37c8e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -265,6 +265,8 @@ QueryCursorImpl> updateSqlFieldsDistributed(String schemaName, PreparedS SqlFieldsQuery fieldsQry, GridQueryCancel cancel) throws IgniteCheckedException { UpdateResult res = updateSqlFields(schemaName, stmt, fieldsQry, false, null, cancel); + checkUpdateResult(res); + QueryCursorImpl> resCur = (QueryCursorImpl>)new QueryCursorImpl(Collections.singletonList (Collections.singletonList(res.cnt)), cancel, false); @@ -404,7 +406,7 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo return doFastUpdate(plan, fieldsQry.getArgs()); } - if (plan.distributed && !loc) + if (plan.distributed && !loc && !fieldsQry.isLocal()) return doDistributedUpdate(schemaName, fieldsQry, plan, cancel); assert !F.isEmpty(plan.selectQry); @@ -1040,12 +1042,13 @@ private static PageProcessingResult processPage(GridCacheContext cctx, * @param fldsQry Query. * @param filter Filter. * @param cancel Cancel state. + * @param local Locality flag. * @return Update result. * @throws IgniteCheckedException if failed. */ UpdateResult mapDistributedUpdate(String schemaName, PreparedStatement stmt, SqlFieldsQuery fldsQry, - IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException { - return updateSqlFields(schemaName, stmt, fldsQry, true, filter, cancel); + IndexingQueryFilter filter, GridQueryCancel cancel, boolean local) throws IgniteCheckedException { + return updateSqlFields(schemaName, stmt, fldsQry, local, filter, cancel); } /** */ @@ -1144,6 +1147,18 @@ static boolean isDmlStatement(Prepared stmt) { return stmt instanceof Merge || stmt instanceof Insert || stmt instanceof Update || stmt instanceof Delete; } + /** */ + static void checkUpdateResult(UpdateResult r) { + if (!F.isEmpty(r.errorKeys())) { + String msg = "Failed to update some keys because they had been modified concurrently " + + "[keys=" + r.errorKeys() + ']'; + + SQLException conEx = createJdbcSqlException(msg, IgniteQueryErrorCode.CONCURRENT_UPDATE); + + throw new IgniteSQLException(conEx); + } + } + /** Result of processing an individual page with {@link IgniteCache#invokeAll} including error details, if any. */ private final static class PageProcessingResult { /** Number of successfully processed items. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 446f17dc79243..d3871e5855de8 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1452,11 +1452,12 @@ UpdateResult runDistributedUpdate( * @param fldsQry Query. * @param filter Filter. * @param cancel Cancel state. + * @param local Locality flag. * @return Update result. * @throws IgniteCheckedException if failed. */ public UpdateResult mapDistributedUpdate(String schemaName, SqlFieldsQuery fldsQry, IndexingQueryFilter filter, - GridQueryCancel cancel) throws IgniteCheckedException { + GridQueryCancel cancel, boolean local) throws IgniteCheckedException { Connection conn = connectionForSchema(schemaName); H2Utils.setupConnection(conn, false, fldsQry.isEnforceJoinOrder()); @@ -1464,7 +1465,7 @@ public UpdateResult mapDistributedUpdate(String schemaName, SqlFieldsQuery fldsQ PreparedStatement stmt = preparedStatementWithParams(conn, fldsQry.getSql(), Arrays.asList(fldsQry.getArgs()), true); - return dmlProc.mapDistributedUpdate(schemaName, stmt, fldsQry, filter, cancel); + return dmlProc.mapDistributedUpdate(schemaName, stmt, fldsQry, filter, cancel, local); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java index e1b8e5836b051..9f8ba8150a160 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java @@ -1514,6 +1514,8 @@ private void splitSelect( // -- SUB-QUERIES boolean hasSubQueries = hasSubQueries(mapQry.where()); + hasSubQueries |= hasSubQueries(mapQry.from()); + for (int i = 0; i < mapQry.columns(false).size(); i++) hasSubQueries |= hasSubQueries(mapQry.column(i)); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index f1e47d3f76d69..664a6c20e415a 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -784,8 +784,41 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th fldsQry.setEnforceJoinOrder(req.isFlagSet(GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER)); fldsQry.setTimeout(req.timeout(), TimeUnit.MILLISECONDS); fldsQry.setPageSize(req.pageSize()); + fldsQry.setLocal(true); - UpdateResult updRes = h2.mapDistributedUpdate(req.schemaName(), fldsQry, filter, cancel); + boolean local = true; + + final boolean replicated = req.isFlagSet(GridH2QueryRequest.FLAG_REPLICATED); + + if (!replicated && !F.isEmpty(cacheIds) && + findFirstPartitioned(cacheIds).config().getQueryParallelism() > 1) { + fldsQry.setDistributedJoins(true); + + local = false; + } + + UpdateResult updRes = h2.mapDistributedUpdate(req.schemaName(), fldsQry, filter, cancel, local); + + GridCacheContext mainCctx = + !F.isEmpty(cacheIds) ? ctx.cache().context().cacheContext(cacheIds.get(0)) : null; + + boolean evt = local && mainCctx != null && ctx.event().isRecordable(EVT_CACHE_QUERY_EXECUTED); + + if (evt) { + ctx.event().record(new CacheQueryExecutedEvent<>( + node, + "SQL query executed.", + EVT_CACHE_QUERY_EXECUTED, + CacheQueryType.SQL.name(), + mainCctx.name(), + null, + req.query(), + null, + null, + req.parameters(), + node.id(), + null)); + } sendUpdateResponse(node, reqId, updRes, null); } @@ -806,6 +839,10 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th } } + private void onDmlRequest0() { + + } + /** * @param node Node. * @param qryReqId Query request ID. @@ -838,6 +875,7 @@ private void sendError(ClusterNode node, long qryReqId, Throwable err) { * @param updResult Update result. * @param error Error message. */ + @SuppressWarnings("deprecation") private void sendUpdateResponse(ClusterNode node, long reqId, UpdateResult updResult, String error) { try { GridH2DmlResponse rsp = new GridH2DmlResponse(reqId, updResult == null ? 0 : updResult.counter(), @@ -848,8 +886,11 @@ private void sendUpdateResponse(ClusterNode node, long reqId, UpdateResult updRe if (node.isLocal()) h2.reduceQueryExecutor().onMessage(ctx.localNodeId(), rsp); - else + else { + rsp.marshall(ctx.config().getMarshaller()); + ctx.io().sendToGridTopic(node, GridTopic.TOPIC_QUERY, rsp, QUERY_POOL); + } } catch (Exception e) { U.error(log, "Failed to send message.", e); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index 9818085feb2ff..b855800146efe 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -896,6 +896,11 @@ public UpdateResult update( final DistributedUpdateRun r = new DistributedUpdateRun(nodes.size(), qryInfo); + int flags = enforceJoinOrder ? GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER : 0; + + if (isReplicatedOnly) + flags |= GridH2QueryRequest.FLAG_REPLICATED; + GridH2DmlRequest req = new GridH2DmlRequest() .requestId(reqId) .topologyVersion(topVer) @@ -905,7 +910,7 @@ public UpdateResult update( .pageSize(pageSize) .parameters(params) .timeout(timeoutMillis) - .flags(enforceJoinOrder ? GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER : 0); + .flags(flags); updRuns.put(reqId, r); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java new file mode 100644 index 0000000000000..7905cf610d81a --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java @@ -0,0 +1,630 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query; + +import java.sql.Date; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import javax.cache.CacheException; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheKeyConfiguration; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cache.affinity.AffinityKeyMapped; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.cache.query.annotations.QuerySqlFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.events.CacheQueryExecutedEvent; +import org.apache.ignite.events.Event; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; + +import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor; +import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jsr166.ThreadLocalRandom8; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED; + +/** Tests for distributed DML. */ +public class IgniteSqlDistributedDmlSelfTest extends GridCommonAbstractTest { + /** IP finder. */ + private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** */ + private static int NODE_COUNT = 4; + + /** */ + private static String NODE_CLIENT = "client"; + + /** */ + private static String CACHE_ORG = "org"; + + /** */ + private static String CACHE_PERSON = "person"; + + /** */ + private static String CACHE_POSITION = "pos"; + + /** */ + private static Ignite client; + + /** */ + private static CountDownLatch waiterLatch; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration c = super.getConfiguration(gridName); + + TcpDiscoverySpi disco = new TcpDiscoverySpi(); + + disco.setIpFinder(IP_FINDER); + + c.setDiscoverySpi(disco); + + List ccfgs = new ArrayList<>(); + + CacheConfiguration ccfg = buildCacheConfiguration(gridName); + + if (ccfg != null) + ccfgs.add(ccfg); + + ccfgs.add(buildCacheConfiguration(CACHE_ORG)); + ccfgs.add(buildCacheConfiguration(CACHE_PERSON)); + ccfgs.add(buildCacheConfiguration(CACHE_POSITION)); + + c.setCacheConfiguration(ccfgs.toArray(new CacheConfiguration[ccfgs.size()])); + + if (gridName.equals(NODE_CLIENT)) + c.setClientMode(true); + + return c; + } + + /** */ + private CacheConfiguration buildCacheConfiguration(String name) { + if (name.equals(CACHE_ORG)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_ORG); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + QueryEntity entity = new QueryEntity(Integer.class, Organization.class); + + ccfg.setQueryEntities(Arrays.asList(entity)); + + ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); + + return ccfg; + } + if (name.equals(CACHE_PERSON)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_PERSON); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + QueryEntity entity = new QueryEntity(PersonKey.class, Person.class); + + ccfg.setQueryEntities(Arrays.asList(entity)); + + ccfg.setKeyConfiguration(new CacheKeyConfiguration(PersonKey.class)); + + ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); + + return ccfg; + } + if (name.equals(CACHE_POSITION)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_POSITION); + + ccfg.setCacheMode(CacheMode.REPLICATED); + + QueryEntity entity = new QueryEntity(Integer.class, Position.class); + + ccfg.setQueryEntities(Arrays.asList(entity)); + + ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); + + return ccfg; + } + + return null; + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startGrids(NODE_COUNT); + + client = startGrid(NODE_CLIENT); + + awaitPartitionMapExchange(); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + checkNoLeaks(); + + super.afterTestsStopped(); + + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + Ignite client = grid(NODE_CLIENT); + + // Stop additional node that is started in one of the test. + stopGrid(NODE_COUNT + 1); + + awaitPartitionMapExchange(); + + client.cache(CACHE_PERSON).clear(); + client.cache(CACHE_ORG).clear(); + client.cache(CACHE_POSITION).clear(); + } + + /** */ + public void testSimpleUpdateDistributedReplicated() throws Exception { + fillCaches(); + + IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_POSITION); + + Position p = cache.get(1); + + List> r = cache.query(new SqlFieldsQuery("UPDATE Position p SET name = CONCAT('A ', name)")).getAll(); + + assertEquals((long)cache.size(), r.get(0).get(0)); + + assertEquals(cache.get(1).name, "A " + p.name); + } + + /** */ + public void testSimpleUpdateDistributedPartitioned() throws Exception { + fillCaches(); + + IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); + + List> r = cache.query(new SqlFieldsQuery( + "UPDATE Person SET position = CASEWHEN(position = 1, 1, position - 1)")).getAll(); + + assertEquals((long)cache.size(), r.get(0).get(0)); + } + + /** */ + public void testDistributedUpdateFailedKeys() throws Exception { + // UPDATE can produce failed keys due to concurrent modification + fillCaches(); + + final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() { + return cache.query(new SqlFieldsQuery("UPDATE Organization SET rate = Modify(_key, rate - 1)")); + } + }, CacheException.class, "Failed to update some keys because they had been modified concurrently"); + } + + /** */ + public void testDistributedUpdateFail() throws Exception { + fillCaches(); + + final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() { + return cache.query(new SqlFieldsQuery("UPDATE Person SET name = Fail(name)")); + } + }, CacheException.class, "Failed to execute SQL query"); + } + + /** */ + public void testQueryParallelism() throws Exception { + String cacheName = CACHE_ORG + "x4"; + + CacheConfiguration cfg = buildCacheConfiguration(CACHE_ORG) + .setQueryParallelism(4) + .setName(cacheName); + + IgniteCache cache = grid(NODE_CLIENT).createCache(cfg); + + for (int i = 0; i < 1024; i++) + cache.put(i, new Organization("Acme Inc #" + i, 0)); + + List> r = cache.query(new SqlFieldsQuery("UPDATE \"" + cacheName + + "\".Organization o SET name = UPPER(name)")).getAll(); + + assertEquals((long)cache.size(), r.get(0).get(0)); + } + + /** */ + public void testEvents() throws Exception { + final CountDownLatch latch = new CountDownLatch(NODE_COUNT); + + final IgnitePredicate pred = new IgnitePredicate() { + @Override public boolean apply(Event evt) { + assert evt instanceof CacheQueryExecutedEvent; + + CacheQueryExecutedEvent qe = (CacheQueryExecutedEvent)evt; + + assertNotNull(qe.clause()); + + latch.countDown(); + + return true; + } + }; + + for (int idx = 0; idx < NODE_COUNT; idx++) + grid(idx).events().localListen(pred, EVT_CACHE_QUERY_EXECUTED); + + IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); + + for (int i = 0; i < 1024; i++) + cache.put(i, new Organization("Acme Inc #" + i, 0)); + + cache.query(new SqlFieldsQuery("UPDATE \"org\".Organization o SET name = UPPER(name)")).getAll(); + + assertTrue(latch.await(5000, MILLISECONDS)); + + for (int idx = 0; idx < NODE_COUNT; idx++) + grid(idx).events().stopLocalListen(pred); + } + + /** */ + public void testSpecificPartitionsUpdate() throws Exception { + fillCaches(); + + Affinity aff = grid(NODE_CLIENT).affinity(CACHE_PERSON); + + int numParts = aff.partitions(); + int parts[] = new int[numParts / 2]; + + for (int idx = 0; idx < numParts / 2; idx++) + parts[idx] = idx * 2; + + IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); + + // UPDATE over even partitions + cache.query(new SqlFieldsQuery("UPDATE Person SET position = 0") + .setPartitions(parts)); + + List> rows = cache.query(new SqlFieldsQuery("SELECT _key, position FROM Person")).getAll(); + + for (List row : rows) { + PersonKey personKey = (PersonKey)row.get(0); + int pos = ((Number)row.get(1)).intValue(); + int part = aff.partition(personKey); + + assertTrue((part % 2 == 0) ^ (pos != 0)); + } + } + + /** */ + public void testCancel() throws Exception { + fillCaches(); + + waiterLatch = new CountDownLatch(NODE_COUNT); + + final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); + + final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { + @Override public Object call() { + return cache.query(new SqlFieldsQuery("UPDATE Organization SET name = WAIT(name)")); + } + }); + + assertTrue(waiterLatch.await(5000, TimeUnit.MILLISECONDS)); + + Collection qCol = grid(NODE_CLIENT).context().query().runningQueries(0); + + assertFalse(qCol.isEmpty()); + + for (GridRunningQueryInfo queryInfo : qCol) + queryInfo.cancel(); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws IgniteCheckedException { + return fut.get(); + } + }, IgniteCheckedException.class, "Future was cancelled"); + } + + /** */ + public void testNodeStopDuringUpdate() throws Exception { + fillCaches(); + + waiterLatch = new CountDownLatch(NODE_COUNT + 1); + + startGrid(NODE_COUNT + 1); + + awaitPartitionMapExchange(); + + final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); + + final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { + @Override public Object call() { + return cache.query(new SqlFieldsQuery("UPDATE Organization SET name = WAIT(name)")); + } + }); + + assertTrue(waiterLatch.await(5000, TimeUnit.MILLISECONDS)); + + stopGrid(NODE_COUNT + 1); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws IgniteCheckedException { + return fut.get(); + } + }, IgniteCheckedException.class, "Update failed because map node left topology"); + } + + /** + * Ensure there are no leaks in data structures associated with distributed dml execution. + */ + private void checkNoLeaks() { + GridQueryProcessor qryProc = grid(NODE_CLIENT).context().query(); + + IgniteH2Indexing h2Idx = GridTestUtils.getFieldValue(qryProc, GridQueryProcessor.class, "idx"); + + GridReduceQueryExecutor rdcQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "rdcQryExec"); + + Map updRuns = GridTestUtils.getFieldValue(rdcQryExec, GridReduceQueryExecutor.class, "updRuns"); + + assertEquals(0, updRuns.size()); + + for (int idx = 0; idx < NODE_COUNT; idx++) { + qryProc = grid(idx).context().query(); + + h2Idx = GridTestUtils.getFieldValue(qryProc, GridQueryProcessor.class, "idx"); + + GridMapQueryExecutor mapQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec"); + + Map qryRess = GridTestUtils.getFieldValue(mapQryExec, GridMapQueryExecutor.class, "qryRess"); + + for (Object obj : qryRess.values()) { + Map updCancels = GridTestUtils.getFieldValue(obj, "updCancels"); + + assertEquals(0, updCancels.size()); + } + } + } + + /** */ + private void fillCaches() { + Ignite client = grid(NODE_CLIENT); + + IgniteCache posCache = client.cache(CACHE_POSITION); + + // Generate positions + Position[] positions = new Position[] { + new Position(1, "High Ranking Officer", 1), + new Position(2, "Administrative worker", 3), + new Position(3, "Worker", 7), + new Position(4, "Security", 2), + new Position(5, "Cleaner", 1) + }; + + for (Position pos: positions) + posCache.put(pos.id, pos); + + // Generate organizations + String[] forms = new String[] {" Inc", " Co", " AG", " Industrees"}; + String[] orgNames = new String[] {"Acme", "Sierra", "Mesa", "Umbrella", "Robotics"}; + String[] names = new String[] {"Mary", "John", "William", "Tom", "Basil", "Ann", "Peter"}; + + IgniteCache personCache = client.cache(CACHE_PERSON); + + IgniteCache orgCache = client.cache(CACHE_ORG); + + int orgId = 0; + int personId = 0; + + for (String orgName : produceCombination(orgNames, orgNames, forms)) { + Organization org = new Organization(orgName, 0); + + orgCache.put(++orgId, org); + + // Generate persons + + List personNames = produceCombination(names, names, new String[]{"s"}); + + int positionId = 0; + int posCounter = 0; + + for (String name : personNames) { + PersonKey pKey = new PersonKey(orgId, ++personId); + + if (positions[positionId].rate < posCounter++) { + posCounter = 0; + positionId = (positionId + 1) % positions.length; + } + + Person person = new Person(name, positions[positionId].id, 0); + + personCache.put(pKey, person); + } + } + } + + /** */ + private List produceCombination(String[] a, String[] b, String[] ends) { + List res = new ArrayList<>(); + + for (int i = 0; i < a.length; i++) { + for (int j = 0; j < b.length; j++) { + String s1 = a[i]; + String s2 = a[j]; + + if (!s1.equals(s2)) { + String end = ends[ThreadLocalRandom8.current().nextInt(ends.length)]; + + res.add(s1 + " " + s2 + end); + } + } + } + + return res; + } + + /** */ + private static class Organization { + /** */ + @QuerySqlField + String name; + + @QuerySqlField + int rate; + + @QuerySqlField + Date updated; + + /** */ + public Organization() { + // No-op. + } + + /** + * @param name Organization name. + */ + public Organization(String name, int rate) { + this.name = name; + this.rate = rate; + this.updated = new Date(System.currentTimeMillis()); + } + } + + /** */ + public static class PersonKey { + /** */ + @AffinityKeyMapped + @QuerySqlField + private Integer orgId; + + /** */ + @QuerySqlField + private Integer id; + + /** */ + PersonKey(int orgId, int id) { + this.orgId = orgId; + this.id = id; + } + } + + /** */ + public static class Person { + /** */ + @QuerySqlField + String name; + + @QuerySqlField + int position; + + @QuerySqlField + int amount; + + @QuerySqlField + Date updated; + + /** */ + private Person(String name, int position, int amount) { + this.name = name; + this.position = position; + this.amount = amount; + this.updated = new Date(System.currentTimeMillis()); + } + } + + /** */ + private static class Position { + /** */ + @QuerySqlField + int id; + + /** */ + @QuerySqlField + String name; + + /** */ + @QuerySqlField + int rate; + + /** */ + public Position(int id, String name, int rate) { + this.id = id; + this.name = name; + this.rate = rate; + } + } + + /** */ + @QuerySqlFunction + public static String Fail(String param) { + throw new IgniteSQLException("Fail() called"); + } + + /** */ + @QuerySqlFunction + public static String Wait(String param) { + try { + waiterLatch.countDown(); + + Thread.sleep(3000); + } + catch (InterruptedException ignore) { + // No-op + } + return param; + } + + /** */ + @QuerySqlFunction + public static int Modify(final int id, final int rate) { + try { + GridTestUtils.runAsync(new Callable() { + @Override public Object call() { + IgniteCache cache = client.cache(CACHE_ORG); + + cache.put(id, new Organization("Acme Inc #" + id, rate + 1)); + + return null; + } + }).get(); + } + catch (Exception e) { + // No-op + } + + return rate - 1; + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java index 5ac0655f953e8..38d2db5787929 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java @@ -120,6 +120,7 @@ import org.apache.ignite.internal.processors.cache.query.IndexingSpiQuerySelfTest; import org.apache.ignite.internal.processors.cache.query.IndexingSpiQueryTxSelfTest; import org.apache.ignite.internal.processors.query.IgniteQueryDedicatedPoolTest; +import org.apache.ignite.internal.processors.query.IgniteSqlDistributedDmlSelfTest; import org.apache.ignite.internal.processors.query.IgniteSqlEntryCacheModeAgnosticTest; import org.apache.ignite.internal.processors.query.IgniteSqlKeyValueFieldsTest; import org.apache.ignite.internal.processors.query.IgniteSqlRoutingTest; @@ -232,6 +233,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgniteCacheInsertSqlQuerySelfTest.class); suite.addTestSuite(IgniteCacheUpdateSqlQuerySelfTest.class); suite.addTestSuite(IgniteCacheDeleteSqlQuerySelfTest.class); + suite.addTestSuite(IgniteSqlDistributedDmlSelfTest.class); suite.addTestSuite(IgniteBinaryObjectQueryArgumentsTest.class); suite.addTestSuite(IgniteBinaryObjectLocalQueryArgumentsTest.class); From bb605e47ba0e544d8392d3d5c2b8c56628feb8d1 Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 8 Sep 2017 15:02:23 +0300 Subject: [PATCH 13/40] Cosmetics. --- .../internal/processors/cache/query/GridCacheSqlQuery.java | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java index 30125869e8970..f38c5b264e33a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java @@ -363,6 +363,7 @@ public boolean hasSubQueries() { /** * @param hasSubQries Flag indicating that query contains sub-queries. + * * @return {@code this}. */ public GridCacheSqlQuery hasSubQueries(boolean hasSubQries) { From 6ae9dbd19bc41043f66de59a2ffa60b2a1b7286d Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 8 Sep 2017 15:02:39 +0300 Subject: [PATCH 14/40] MInor optimization to GridSqlQuerySplitter. --- .../query/h2/sql/GridSqlQuerySplitter.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java index 9f8ba8150a160..8bf102a92f784 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java @@ -1512,12 +1512,17 @@ private void splitSelect( } // -- SUB-QUERIES - boolean hasSubQueries = hasSubQueries(mapQry.where()); + boolean hasSubQueries = hasSubQueries(mapQry.where()) || hasSubQueries(mapQry.from()); - hasSubQueries |= hasSubQueries(mapQry.from()); + if (!hasSubQueries) { + for (int i = 0; i < mapQry.columns(false).size(); i++) { + if (hasSubQueries(mapQry.column(i))) { + hasSubQueries = true; - for (int i = 0; i < mapQry.columns(false).size(); i++) - hasSubQueries |= hasSubQueries(mapQry.column(i)); + break; + } + } + } // Replace the given select with generated reduce query in the parent. prnt.child(childIdx, rdcQry); From 47ddd662a4abeef187e70578b84c577d83cfadf3 Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 8 Sep 2017 15:11:59 +0300 Subject: [PATCH 15/40] Removing warnings in test. --- .../query/IgniteSqlDistributedDmlSelfTest.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java index 7905cf610d81a..72b36510b4533 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java @@ -19,8 +19,8 @@ import java.sql.Date; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; @@ -58,6 +58,7 @@ import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED; /** Tests for distributed DML. */ +@SuppressWarnings({"unchecked", "ThrowableResultOfMethodCallIgnored"}) public class IgniteSqlDistributedDmlSelfTest extends GridCommonAbstractTest { /** IP finder. */ private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); @@ -121,7 +122,7 @@ private CacheConfiguration buildCacheConfiguration(String name) { QueryEntity entity = new QueryEntity(Integer.class, Organization.class); - ccfg.setQueryEntities(Arrays.asList(entity)); + ccfg.setQueryEntities(Collections.singletonList(entity)); ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); @@ -134,7 +135,7 @@ private CacheConfiguration buildCacheConfiguration(String name) { QueryEntity entity = new QueryEntity(PersonKey.class, Person.class); - ccfg.setQueryEntities(Arrays.asList(entity)); + ccfg.setQueryEntities(Collections.singletonList(entity)); ccfg.setKeyConfiguration(new CacheKeyConfiguration(PersonKey.class)); @@ -149,13 +150,15 @@ private CacheConfiguration buildCacheConfiguration(String name) { QueryEntity entity = new QueryEntity(Integer.class, Position.class); - ccfg.setQueryEntities(Arrays.asList(entity)); + ccfg.setQueryEntities(Collections.singletonList(entity)); ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); return ccfg; } + assert false; + return null; } From d98aeef392ddb8ba53d25f3bf8969356773f241d Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Tue, 12 Sep 2017 17:12:16 +0300 Subject: [PATCH 16/40] IGNITE-6024: fixed test --- .../processors/query/IgniteSqlDistributedDmlSelfTest.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java index 72b36510b4533..d9facae4158b1 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java @@ -96,11 +96,6 @@ public class IgniteSqlDistributedDmlSelfTest extends GridCommonAbstractTest { List ccfgs = new ArrayList<>(); - CacheConfiguration ccfg = buildCacheConfiguration(gridName); - - if (ccfg != null) - ccfgs.add(ccfg); - ccfgs.add(buildCacheConfiguration(CACHE_ORG)); ccfgs.add(buildCacheConfiguration(CACHE_PERSON)); ccfgs.add(buildCacheConfiguration(CACHE_POSITION)); From bb2cac1617548e008fa5410299bc02e678323e53 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Wed, 13 Sep 2017 14:21:43 +0300 Subject: [PATCH 17/40] IGNITE-6024: fixed flaky test --- .../IgniteSqlDistributedDmlSelfTest.java | 58 ++++++++++++------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java index d9facae4158b1..f99c35a991cbb 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import javax.cache.CacheException; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; @@ -47,6 +46,7 @@ import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor; import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor; +import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.lang.IgnitePredicate; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; @@ -82,7 +82,7 @@ public class IgniteSqlDistributedDmlSelfTest extends GridCommonAbstractTest { private static Ignite client; /** */ - private static CountDownLatch waiterLatch; + private static CountDownLatch latch; /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { @@ -246,6 +246,7 @@ public void testDistributedUpdateFail() throws Exception { } /** */ + @SuppressWarnings("ConstantConditions") public void testQueryParallelism() throws Exception { String cacheName = CACHE_ORG + "x4"; @@ -331,8 +332,6 @@ public void testSpecificPartitionsUpdate() throws Exception { public void testCancel() throws Exception { fillCaches(); - waiterLatch = new CountDownLatch(NODE_COUNT); - final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { @@ -341,14 +340,20 @@ public void testCancel() throws Exception { } }); - assertTrue(waiterLatch.await(5000, TimeUnit.MILLISECONDS)); + GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + Collection qCol = + grid(NODE_CLIENT).context().query().runningQueries(0); - Collection qCol = grid(NODE_CLIENT).context().query().runningQueries(0); + if (qCol.isEmpty()) + return false; - assertFalse(qCol.isEmpty()); + for (GridRunningQueryInfo queryInfo : qCol) + queryInfo.cancel(); - for (GridRunningQueryInfo queryInfo : qCol) - queryInfo.cancel(); + return true; + } + }, 5000); GridTestUtils.assertThrows(log, new Callable() { @Override public Object call() throws IgniteCheckedException { @@ -359,23 +364,23 @@ public void testCancel() throws Exception { /** */ public void testNodeStopDuringUpdate() throws Exception { - fillCaches(); - - waiterLatch = new CountDownLatch(NODE_COUNT + 1); - startGrid(NODE_COUNT + 1); awaitPartitionMapExchange(); + fillCaches(); + + latch = new CountDownLatch(NODE_COUNT + 1); + final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { @Override public Object call() { - return cache.query(new SqlFieldsQuery("UPDATE Organization SET name = WAIT(name)")); + return cache.query(new SqlFieldsQuery("UPDATE Organization SET name = COUNTANDWAIT(name)")); } }); - assertTrue(waiterLatch.await(5000, TimeUnit.MILLISECONDS)); + assertTrue(latch.await(5000, MILLISECONDS)); stopGrid(NODE_COUNT + 1); @@ -436,7 +441,7 @@ private void fillCaches() { posCache.put(pos.id, pos); // Generate organizations - String[] forms = new String[] {" Inc", " Co", " AG", " Industrees"}; + String[] forms = new String[] {" Inc", " Co", " AG", " Industries"}; String[] orgNames = new String[] {"Acme", "Sierra", "Mesa", "Umbrella", "Robotics"}; String[] names = new String[] {"Mary", "John", "William", "Tom", "Basil", "Ann", "Peter"}; @@ -478,11 +483,8 @@ private void fillCaches() { private List produceCombination(String[] a, String[] b, String[] ends) { List res = new ArrayList<>(); - for (int i = 0; i < a.length; i++) { - for (int j = 0; j < b.length; j++) { - String s1 = a[i]; - String s2 = a[j]; - + for (String s1 : a) { + for (String s2 : b) { if (!s1.equals(s2)) { String end = ends[ThreadLocalRandom8.current().nextInt(ends.length)]; @@ -595,7 +597,19 @@ public static String Fail(String param) { @QuerySqlFunction public static String Wait(String param) { try { - waiterLatch.countDown(); + Thread.sleep(3000); + } + catch (InterruptedException ignore) { + // No-op + } + return param; + } + + /** */ + @QuerySqlFunction + public static String CountAndWait(String param) { + try { + latch.countDown(); Thread.sleep(3000); } From 662bb71bb2a6748777f8a8e83541fba754aaf35d Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Mon, 2 Oct 2017 19:18:13 +0300 Subject: [PATCH 18/40] IGNITE-6024: Fixing review comments --- .../query/h2/DmlStatementsProcessor.java | 14 ++++-- .../processors/query/h2/UpdateResult.java | 13 ++++-- .../h2/twostep/DistributedUpdateRun.java | 46 +++++++++++-------- .../h2/twostep/GridMapQueryExecutor.java | 4 -- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 7b106d9089367..12ae9ecd5669b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -199,8 +199,8 @@ else if (!opCtx.isKeepBinary()) cctx.operationContextPerCall(opCtx); } - items += r.cnt; - errKeys = r.errKeys; + items += r.counter(); + errKeys = r.errorKeys(); if (F.isEmpty(errKeys)) break; @@ -270,7 +270,7 @@ QueryCursorImpl> updateSqlFieldsDistributed(String schemaName, Connectio checkUpdateResult(res); QueryCursorImpl> resCur = (QueryCursorImpl>)new QueryCursorImpl(Collections.singletonList - (Collections.singletonList(res.cnt)), cancel, false); + (Collections.singletonList(res.counter())), cancel, false); resCur.fieldsMeta(UPDATE_RESULT_META); @@ -297,7 +297,7 @@ GridQueryFieldsResult updateSqlFieldsLocal(String schemaName, Connection conn, P filters, cancel); return new GridQueryFieldsResultAdapter(UPDATE_RESULT_META, - new IgniteSingletonIterator(Collections.singletonList(res.cnt))); + new IgniteSingletonIterator(Collections.singletonList(res.counter()))); } /** @@ -1180,7 +1180,11 @@ static boolean isDmlStatement(Prepared stmt) { return stmt instanceof Merge || stmt instanceof Insert || stmt instanceof Update || stmt instanceof Delete; } - /** */ + /** + * Check update result for erroneous keys and throws concurrent update exception if necessary. + * + * @param r Update result. + */ static void checkUpdateResult(UpdateResult r) { if (!F.isEmpty(r.errorKeys())) { String msg = "Failed to update some keys because they had been modified concurrently " + diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java index 1ab368d15e8da..8c2fc7e569615 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java @@ -19,7 +19,6 @@ import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; -import org.jetbrains.annotations.NotNull; /** Update result - modifications count and keys to re-run query with, if needed. */ public final class UpdateResult { @@ -30,13 +29,17 @@ public final class UpdateResult { final static UpdateResult ZERO = new UpdateResult(0, X.EMPTY_OBJECT_ARRAY); /** Number of processed items. */ - final long cnt; + private final long cnt; /** Keys that failed to be updated or deleted due to concurrent modification of values. */ - @NotNull - final Object[] errKeys; + private final Object[] errKeys; - /** */ + /** + * Constructor. + * + * @param cnt Updated rows count. + * @param errKeys Array of erroneous keys. + */ public @SuppressWarnings("ConstantConditions") UpdateResult(long cnt, Object[] errKeys) { this.cnt = cnt; this.errKeys = U.firstNotNull(errKeys, X.EMPTY_OBJECT_ARRAY); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java index 2a26aa775fd58..5cc4c05b86e55 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java @@ -18,14 +18,14 @@ package org.apache.ignite.internal.processors.query.h2.twostep; import java.util.Arrays; +import java.util.HashSet; +import java.util.List; import java.util.UUID; -import java.util.concurrent.atomic.AtomicLong; import javax.cache.CacheException; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.processors.query.GridRunningQueryInfo; import org.apache.ignite.internal.processors.query.h2.UpdateResult; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2DmlResponse; -import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.typedef.F; @@ -35,13 +35,13 @@ class DistributedUpdateRun { private final int nodeCount; /** Registers nodes that have responded. */ - private final GridConcurrentHashSet rspNodes; + private final HashSet rspNodes; /** Accumulates total number of updated rows. */ - private final AtomicLong updCntr = new AtomicLong(); + private long updCntr = 0L; /** Accumulates error keys. */ - private final GridConcurrentHashSet errorKeys = new GridConcurrentHashSet<>(); + private HashSet errorKeys; /** Query info. */ private final GridRunningQueryInfo qry; @@ -59,7 +59,7 @@ class DistributedUpdateRun { this.nodeCount = nodeCount; this.qry = qry; - rspNodes = new GridConcurrentHashSet<>(nodeCount); + rspNodes = new HashSet<>(nodeCount); } /** @@ -100,24 +100,32 @@ void handleNodeLeft(UUID nodeId) { * @param msg Response message. */ void handleResponse(UUID id, GridH2DmlResponse msg) { - if (!rspNodes.add(id)) - return; // ignore duplicated messages + synchronized (this) { + if (!rspNodes.add(id)) + return; // ignore duplicated messages - String err = msg.error(); + String err = msg.error(); - if (err != null) { - fut.onDone(new IgniteCheckedException("Update failed. " + (F.isEmpty(err)? "" : err) + "[reqId=" + - msg.requestId() + ", node=" + id + "].")); + if (err != null) { + fut.onDone(new IgniteCheckedException("Update failed. " + (F.isEmpty(err) ? "" : err) + "[reqId=" + + msg.requestId() + ", node=" + id + "].")); - return; - } + return; + } + + if (!F.isEmpty(msg.errorKeys())) { + List errList = Arrays.asList(msg.errorKeys()); - if (!F.isEmpty(msg.errorKeys())) - errorKeys.addAll(Arrays.asList(msg.errorKeys())); + if (errorKeys == null) + errorKeys = new HashSet<>(errList); + else + errorKeys.addAll(errList); + } - long cntr = updCntr.addAndGet(msg.updateCounter()); + updCntr += msg.updateCounter(); - if (rspNodes.size() == nodeCount) - fut.onDone(new UpdateResult(cntr, errorKeys.toArray())); + if (rspNodes.size() == nodeCount) + fut.onDone(new UpdateResult(updCntr, errorKeys.toArray())); + } } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 664a6c20e415a..77b928f062e27 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -839,10 +839,6 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th } } - private void onDmlRequest0() { - - } - /** * @param node Node. * @param qryReqId Query request ID. From 2fb7293ab403e9aae888cc3760eba07603e2355b Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Mon, 2 Oct 2017 20:34:44 +0300 Subject: [PATCH 19/40] IGNITE-6024: Applied review comments --- .../processors/query/h2/DmlStatementsProcessor.java | 9 +++++++-- .../query/h2/twostep/GridReduceQueryExecutor.java | 11 ++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 12ae9ecd5669b..3709cb9cab7fc 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -413,8 +413,13 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo return doFastUpdate(plan, fieldsQry.getArgs()); } - if (plan.distributed && !loc && !fieldsQry.isLocal()) - return doDistributedUpdate(schemaName, fieldsQry, plan, cancel); + if (plan.distributed && !loc && !fieldsQry.isLocal()) { + UpdateResult result = doDistributedUpdate(schemaName, fieldsQry, plan, cancel); + + // null is returned in case not all nodes support distributed DML. + if (result != null) + return result; + } assert !F.isEmpty(plan.selectQry); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java index b855800146efe..f85cd94a10d62 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java @@ -856,7 +856,7 @@ public Iterator> query( * @param parts Partitions. * @param isReplicatedOnly Whether query uses only replicated caches. * @param cancel Cancel state. - * @return Update result. + * @return Update result, or {@code null} when some map node doesn't support distributed DML. */ public UpdateResult update( String schemaName, @@ -894,6 +894,15 @@ public UpdateResult update( nodes = singletonList(F.rand(nodes)); } + for (ClusterNode n : nodes) { + if (!n.version().greaterThanEqual(2, 3, 0)) { + log.warning("Server-side DML optimization is skipped because map node does not support it. " + + "Falling back to normal DML. [node=" + n.id() + ", v=" + n.version() + "]."); + + return null; + } + } + final DistributedUpdateRun r = new DistributedUpdateRun(nodes.size(), qryInfo); int flags = enforceJoinOrder ? GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER : 0; From 61ec061b68f1818dc1e4ff00b5da525290280f90 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Mon, 2 Oct 2017 20:41:20 +0300 Subject: [PATCH 20/40] IGNITE-6024: fix --- .../processors/query/h2/twostep/DistributedUpdateRun.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java index 5cc4c05b86e55..ffd75adc16af3 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java @@ -125,7 +125,7 @@ void handleResponse(UUID id, GridH2DmlResponse msg) { updCntr += msg.updateCounter(); if (rspNodes.size() == nodeCount) - fut.onDone(new UpdateResult(updCntr, errorKeys.toArray())); + fut.onDone(new UpdateResult(updCntr, errorKeys == null ? null : errorKeys.toArray())); } } } From fc0b5ee62ed21511974f717512360f684e662638 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Wed, 4 Oct 2017 19:04:29 +0300 Subject: [PATCH 21/40] IGNITE-6024: introduced updateOnServer flag --- .../ignite/cache/query/SqlFieldsQuery.java | 40 +++++++++++++++++++ .../query/h2/DmlStatementsProcessor.java | 4 +- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/cache/query/SqlFieldsQuery.java b/modules/core/src/main/java/org/apache/ignite/cache/query/SqlFieldsQuery.java index 2d128d158b9fb..50762f2c79571 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/query/SqlFieldsQuery.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/query/SqlFieldsQuery.java @@ -74,6 +74,9 @@ public class SqlFieldsQuery extends Query> { /** */ private boolean lazy; + /** Whether server side DML should be enabled. */ + private boolean updateOnServer; + /** Partitions for query */ private int[] parts; @@ -94,6 +97,7 @@ public SqlFieldsQuery(SqlFieldsQuery qry) { distributedJoins = qry.distributedJoins; replicatedOnly = qry.replicatedOnly; lazy = qry.lazy; + updateOnServer = qry.updateOnServer; parts = qry.parts; schema = qry.schema; } @@ -322,6 +326,42 @@ public boolean isLazy() { return lazy; } + /** + * Sets server side update flag. + *

+ * By default, when processing DML command, Ignite first fetches all affected intermediate rows for analysis to the + * node which initiated the query and only then forms batches of updated values to be send to remote nodes. + * For simple DML commands (that however affect great deal of rows) such approach may be an overkill in terms of + * network delays and memory usage on initiating node. Use this flag as hint for Ignite to do all intermediate rows + * analysis and updates in place on corresponding remote data nodes. + *

+ * There are limitations to what DML command can be optimized this way. The command containing LIMIT, OFFSET, + * DISTINCT, ORDER BY, GROUP BY, sub-query or UNION will be processed the usual way despite this flag setting. + *

+ * Defaults to {@code false}, meaning that intermediate results will be fetched to initiating node first. + * Only affects DML commands. Ignored when {@link #isLocal()} is {@code true}. + * Note that when set to {@code true}, the query may fail in the case of even single node failure. + * + * @param updateOnServer + * @return {@code this} For chaining. + */ + public SqlFieldsQuery setUpdateOnServer(boolean updateOnServer) { + this.updateOnServer = updateOnServer; + + return this; + } + + /** + * Gets server side update flag. + *

+ * See {@link #setUpdateOnServer(boolean)} for more information. + * + * @return Server side update flag. + */ + public boolean isUpdateOnServer() { + return updateOnServer; + } + /** * Gets partitions for query, in ascending order. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 3709cb9cab7fc..66bf261241ee9 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -413,7 +413,7 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo return doFastUpdate(plan, fieldsQry.getArgs()); } - if (plan.distributed && !loc && !fieldsQry.isLocal()) { + if (plan.distributed && !loc && !fieldsQry.isLocal() && fieldsQry.isUpdateOnServer()) { UpdateResult result = doDistributedUpdate(schemaName, fieldsQry, plan, cancel); // null is returned in case not all nodes support distributed DML. @@ -512,7 +512,7 @@ private UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared res = UpdatePlanBuilder.planForStatement(p, errKeysPos); - if (!loc && !F.isEmpty(res.selectQry)) + if (fieldsQry.isUpdateOnServer() && !loc && !F.isEmpty(res.selectQry)) checkPlanCanBeDistributed(fieldsQry, conn, res); // Don't cache re-runs From 25d031987b9d79040afc7d7afcc812d88be7e33b Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Thu, 5 Oct 2017 20:14:23 +0300 Subject: [PATCH 22/40] IGNITE-6024: wip --- .../IgniteSqlDistributedDmlSelfTest.java | 1617 ++++++++++------- 1 file changed, 975 insertions(+), 642 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java index f99c35a991cbb..dba0283d76d90 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java @@ -1,642 +1,975 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query; - -import java.sql.Date; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import javax.cache.CacheException; -import org.apache.ignite.Ignite; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.cache.CacheKeyConfiguration; -import org.apache.ignite.cache.CacheMode; -import org.apache.ignite.cache.QueryEntity; -import org.apache.ignite.cache.affinity.Affinity; -import org.apache.ignite.cache.affinity.AffinityKeyMapped; -import org.apache.ignite.cache.query.SqlFieldsQuery; -import org.apache.ignite.cache.query.annotations.QuerySqlField; -import org.apache.ignite.cache.query.annotations.QuerySqlFunction; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.events.CacheQueryExecutedEvent; -import org.apache.ignite.events.Event; -import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; - -import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor; -import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor; -import org.apache.ignite.internal.util.lang.GridAbsPredicate; -import org.apache.ignite.lang.IgnitePredicate; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.ignite.testframework.GridTestUtils; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -import org.jsr166.ThreadLocalRandom8; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED; - -/** Tests for distributed DML. */ -@SuppressWarnings({"unchecked", "ThrowableResultOfMethodCallIgnored"}) -public class IgniteSqlDistributedDmlSelfTest extends GridCommonAbstractTest { - /** IP finder. */ - private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); - - /** */ - private static int NODE_COUNT = 4; - - /** */ - private static String NODE_CLIENT = "client"; - - /** */ - private static String CACHE_ORG = "org"; - - /** */ - private static String CACHE_PERSON = "person"; - - /** */ - private static String CACHE_POSITION = "pos"; - - /** */ - private static Ignite client; - - /** */ - private static CountDownLatch latch; - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { - IgniteConfiguration c = super.getConfiguration(gridName); - - TcpDiscoverySpi disco = new TcpDiscoverySpi(); - - disco.setIpFinder(IP_FINDER); - - c.setDiscoverySpi(disco); - - List ccfgs = new ArrayList<>(); - - ccfgs.add(buildCacheConfiguration(CACHE_ORG)); - ccfgs.add(buildCacheConfiguration(CACHE_PERSON)); - ccfgs.add(buildCacheConfiguration(CACHE_POSITION)); - - c.setCacheConfiguration(ccfgs.toArray(new CacheConfiguration[ccfgs.size()])); - - if (gridName.equals(NODE_CLIENT)) - c.setClientMode(true); - - return c; - } - - /** */ - private CacheConfiguration buildCacheConfiguration(String name) { - if (name.equals(CACHE_ORG)) { - CacheConfiguration ccfg = new CacheConfiguration(CACHE_ORG); - - ccfg.setCacheMode(CacheMode.PARTITIONED); - - QueryEntity entity = new QueryEntity(Integer.class, Organization.class); - - ccfg.setQueryEntities(Collections.singletonList(entity)); - - ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); - - return ccfg; - } - if (name.equals(CACHE_PERSON)) { - CacheConfiguration ccfg = new CacheConfiguration(CACHE_PERSON); - - ccfg.setCacheMode(CacheMode.PARTITIONED); - - QueryEntity entity = new QueryEntity(PersonKey.class, Person.class); - - ccfg.setQueryEntities(Collections.singletonList(entity)); - - ccfg.setKeyConfiguration(new CacheKeyConfiguration(PersonKey.class)); - - ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); - - return ccfg; - } - if (name.equals(CACHE_POSITION)) { - CacheConfiguration ccfg = new CacheConfiguration(CACHE_POSITION); - - ccfg.setCacheMode(CacheMode.REPLICATED); - - QueryEntity entity = new QueryEntity(Integer.class, Position.class); - - ccfg.setQueryEntities(Collections.singletonList(entity)); - - ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); - - return ccfg; - } - - assert false; - - return null; - } - - /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - super.beforeTestsStarted(); - - startGrids(NODE_COUNT); - - client = startGrid(NODE_CLIENT); - - awaitPartitionMapExchange(); - } - - /** {@inheritDoc} */ - @Override protected void afterTestsStopped() throws Exception { - checkNoLeaks(); - - super.afterTestsStopped(); - - stopAllGrids(); - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - Ignite client = grid(NODE_CLIENT); - - // Stop additional node that is started in one of the test. - stopGrid(NODE_COUNT + 1); - - awaitPartitionMapExchange(); - - client.cache(CACHE_PERSON).clear(); - client.cache(CACHE_ORG).clear(); - client.cache(CACHE_POSITION).clear(); - } - - /** */ - public void testSimpleUpdateDistributedReplicated() throws Exception { - fillCaches(); - - IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_POSITION); - - Position p = cache.get(1); - - List> r = cache.query(new SqlFieldsQuery("UPDATE Position p SET name = CONCAT('A ', name)")).getAll(); - - assertEquals((long)cache.size(), r.get(0).get(0)); - - assertEquals(cache.get(1).name, "A " + p.name); - } - - /** */ - public void testSimpleUpdateDistributedPartitioned() throws Exception { - fillCaches(); - - IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); - - List> r = cache.query(new SqlFieldsQuery( - "UPDATE Person SET position = CASEWHEN(position = 1, 1, position - 1)")).getAll(); - - assertEquals((long)cache.size(), r.get(0).get(0)); - } - - /** */ - public void testDistributedUpdateFailedKeys() throws Exception { - // UPDATE can produce failed keys due to concurrent modification - fillCaches(); - - final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); - - GridTestUtils.assertThrows(log, new Callable() { - @Override public Object call() { - return cache.query(new SqlFieldsQuery("UPDATE Organization SET rate = Modify(_key, rate - 1)")); - } - }, CacheException.class, "Failed to update some keys because they had been modified concurrently"); - } - - /** */ - public void testDistributedUpdateFail() throws Exception { - fillCaches(); - - final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); - - GridTestUtils.assertThrows(log, new Callable() { - @Override public Object call() { - return cache.query(new SqlFieldsQuery("UPDATE Person SET name = Fail(name)")); - } - }, CacheException.class, "Failed to execute SQL query"); - } - - /** */ - @SuppressWarnings("ConstantConditions") - public void testQueryParallelism() throws Exception { - String cacheName = CACHE_ORG + "x4"; - - CacheConfiguration cfg = buildCacheConfiguration(CACHE_ORG) - .setQueryParallelism(4) - .setName(cacheName); - - IgniteCache cache = grid(NODE_CLIENT).createCache(cfg); - - for (int i = 0; i < 1024; i++) - cache.put(i, new Organization("Acme Inc #" + i, 0)); - - List> r = cache.query(new SqlFieldsQuery("UPDATE \"" + cacheName + - "\".Organization o SET name = UPPER(name)")).getAll(); - - assertEquals((long)cache.size(), r.get(0).get(0)); - } - - /** */ - public void testEvents() throws Exception { - final CountDownLatch latch = new CountDownLatch(NODE_COUNT); - - final IgnitePredicate pred = new IgnitePredicate() { - @Override public boolean apply(Event evt) { - assert evt instanceof CacheQueryExecutedEvent; - - CacheQueryExecutedEvent qe = (CacheQueryExecutedEvent)evt; - - assertNotNull(qe.clause()); - - latch.countDown(); - - return true; - } - }; - - for (int idx = 0; idx < NODE_COUNT; idx++) - grid(idx).events().localListen(pred, EVT_CACHE_QUERY_EXECUTED); - - IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); - - for (int i = 0; i < 1024; i++) - cache.put(i, new Organization("Acme Inc #" + i, 0)); - - cache.query(new SqlFieldsQuery("UPDATE \"org\".Organization o SET name = UPPER(name)")).getAll(); - - assertTrue(latch.await(5000, MILLISECONDS)); - - for (int idx = 0; idx < NODE_COUNT; idx++) - grid(idx).events().stopLocalListen(pred); - } - - /** */ - public void testSpecificPartitionsUpdate() throws Exception { - fillCaches(); - - Affinity aff = grid(NODE_CLIENT).affinity(CACHE_PERSON); - - int numParts = aff.partitions(); - int parts[] = new int[numParts / 2]; - - for (int idx = 0; idx < numParts / 2; idx++) - parts[idx] = idx * 2; - - IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); - - // UPDATE over even partitions - cache.query(new SqlFieldsQuery("UPDATE Person SET position = 0") - .setPartitions(parts)); - - List> rows = cache.query(new SqlFieldsQuery("SELECT _key, position FROM Person")).getAll(); - - for (List row : rows) { - PersonKey personKey = (PersonKey)row.get(0); - int pos = ((Number)row.get(1)).intValue(); - int part = aff.partition(personKey); - - assertTrue((part % 2 == 0) ^ (pos != 0)); - } - } - - /** */ - public void testCancel() throws Exception { - fillCaches(); - - final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); - - final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { - @Override public Object call() { - return cache.query(new SqlFieldsQuery("UPDATE Organization SET name = WAIT(name)")); - } - }); - - GridTestUtils.waitForCondition(new GridAbsPredicate() { - @Override public boolean apply() { - Collection qCol = - grid(NODE_CLIENT).context().query().runningQueries(0); - - if (qCol.isEmpty()) - return false; - - for (GridRunningQueryInfo queryInfo : qCol) - queryInfo.cancel(); - - return true; - } - }, 5000); - - GridTestUtils.assertThrows(log, new Callable() { - @Override public Object call() throws IgniteCheckedException { - return fut.get(); - } - }, IgniteCheckedException.class, "Future was cancelled"); - } - - /** */ - public void testNodeStopDuringUpdate() throws Exception { - startGrid(NODE_COUNT + 1); - - awaitPartitionMapExchange(); - - fillCaches(); - - latch = new CountDownLatch(NODE_COUNT + 1); - - final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); - - final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { - @Override public Object call() { - return cache.query(new SqlFieldsQuery("UPDATE Organization SET name = COUNTANDWAIT(name)")); - } - }); - - assertTrue(latch.await(5000, MILLISECONDS)); - - stopGrid(NODE_COUNT + 1); - - GridTestUtils.assertThrows(log, new Callable() { - @Override public Object call() throws IgniteCheckedException { - return fut.get(); - } - }, IgniteCheckedException.class, "Update failed because map node left topology"); - } - - /** - * Ensure there are no leaks in data structures associated with distributed dml execution. - */ - private void checkNoLeaks() { - GridQueryProcessor qryProc = grid(NODE_CLIENT).context().query(); - - IgniteH2Indexing h2Idx = GridTestUtils.getFieldValue(qryProc, GridQueryProcessor.class, "idx"); - - GridReduceQueryExecutor rdcQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "rdcQryExec"); - - Map updRuns = GridTestUtils.getFieldValue(rdcQryExec, GridReduceQueryExecutor.class, "updRuns"); - - assertEquals(0, updRuns.size()); - - for (int idx = 0; idx < NODE_COUNT; idx++) { - qryProc = grid(idx).context().query(); - - h2Idx = GridTestUtils.getFieldValue(qryProc, GridQueryProcessor.class, "idx"); - - GridMapQueryExecutor mapQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec"); - - Map qryRess = GridTestUtils.getFieldValue(mapQryExec, GridMapQueryExecutor.class, "qryRess"); - - for (Object obj : qryRess.values()) { - Map updCancels = GridTestUtils.getFieldValue(obj, "updCancels"); - - assertEquals(0, updCancels.size()); - } - } - } - - /** */ - private void fillCaches() { - Ignite client = grid(NODE_CLIENT); - - IgniteCache posCache = client.cache(CACHE_POSITION); - - // Generate positions - Position[] positions = new Position[] { - new Position(1, "High Ranking Officer", 1), - new Position(2, "Administrative worker", 3), - new Position(3, "Worker", 7), - new Position(4, "Security", 2), - new Position(5, "Cleaner", 1) - }; - - for (Position pos: positions) - posCache.put(pos.id, pos); - - // Generate organizations - String[] forms = new String[] {" Inc", " Co", " AG", " Industries"}; - String[] orgNames = new String[] {"Acme", "Sierra", "Mesa", "Umbrella", "Robotics"}; - String[] names = new String[] {"Mary", "John", "William", "Tom", "Basil", "Ann", "Peter"}; - - IgniteCache personCache = client.cache(CACHE_PERSON); - - IgniteCache orgCache = client.cache(CACHE_ORG); - - int orgId = 0; - int personId = 0; - - for (String orgName : produceCombination(orgNames, orgNames, forms)) { - Organization org = new Organization(orgName, 0); - - orgCache.put(++orgId, org); - - // Generate persons - - List personNames = produceCombination(names, names, new String[]{"s"}); - - int positionId = 0; - int posCounter = 0; - - for (String name : personNames) { - PersonKey pKey = new PersonKey(orgId, ++personId); - - if (positions[positionId].rate < posCounter++) { - posCounter = 0; - positionId = (positionId + 1) % positions.length; - } - - Person person = new Person(name, positions[positionId].id, 0); - - personCache.put(pKey, person); - } - } - } - - /** */ - private List produceCombination(String[] a, String[] b, String[] ends) { - List res = new ArrayList<>(); - - for (String s1 : a) { - for (String s2 : b) { - if (!s1.equals(s2)) { - String end = ends[ThreadLocalRandom8.current().nextInt(ends.length)]; - - res.add(s1 + " " + s2 + end); - } - } - } - - return res; - } - - /** */ - private static class Organization { - /** */ - @QuerySqlField - String name; - - @QuerySqlField - int rate; - - @QuerySqlField - Date updated; - - /** */ - public Organization() { - // No-op. - } - - /** - * @param name Organization name. - */ - public Organization(String name, int rate) { - this.name = name; - this.rate = rate; - this.updated = new Date(System.currentTimeMillis()); - } - } - - /** */ - public static class PersonKey { - /** */ - @AffinityKeyMapped - @QuerySqlField - private Integer orgId; - - /** */ - @QuerySqlField - private Integer id; - - /** */ - PersonKey(int orgId, int id) { - this.orgId = orgId; - this.id = id; - } - } - - /** */ - public static class Person { - /** */ - @QuerySqlField - String name; - - @QuerySqlField - int position; - - @QuerySqlField - int amount; - - @QuerySqlField - Date updated; - - /** */ - private Person(String name, int position, int amount) { - this.name = name; - this.position = position; - this.amount = amount; - this.updated = new Date(System.currentTimeMillis()); - } - } - - /** */ - private static class Position { - /** */ - @QuerySqlField - int id; - - /** */ - @QuerySqlField - String name; - - /** */ - @QuerySqlField - int rate; - - /** */ - public Position(int id, String name, int rate) { - this.id = id; - this.name = name; - this.rate = rate; - } - } - - /** */ - @QuerySqlFunction - public static String Fail(String param) { - throw new IgniteSQLException("Fail() called"); - } - - /** */ - @QuerySqlFunction - public static String Wait(String param) { - try { - Thread.sleep(3000); - } - catch (InterruptedException ignore) { - // No-op - } - return param; - } - - /** */ - @QuerySqlFunction - public static String CountAndWait(String param) { - try { - latch.countDown(); - - Thread.sleep(3000); - } - catch (InterruptedException ignore) { - // No-op - } - return param; - } - - /** */ - @QuerySqlFunction - public static int Modify(final int id, final int rate) { - try { - GridTestUtils.runAsync(new Callable() { - @Override public Object call() { - IgniteCache cache = client.cache(CACHE_ORG); - - cache.put(id, new Organization("Acme Inc #" + id, rate + 1)); - - return null; - } - }).get(); - } - catch (Exception e) { - // No-op - } - - return rate - 1; - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query; + +import java.sql.Date; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import javax.cache.CacheException; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheKeyConfiguration; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cache.affinity.AffinityKeyMapped; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.cache.query.annotations.QuerySqlFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.events.CacheQueryExecutedEvent; +import org.apache.ignite.events.Event; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; +import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor; +import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor; +import org.apache.ignite.internal.util.lang.GridAbsPredicate; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jsr166.ThreadLocalRandom8; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED; + +/** Tests for distributed DML. */ +@SuppressWarnings({"unchecked", "ThrowableResultOfMethodCallIgnored"}) +public class IgniteSqlDistributedDmlSelfTest extends GridCommonAbstractTest { + /** IP finder. */ + private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** */ + private static int NODE_COUNT = 4; + + /** */ + private static String NODE_CLIENT = "client"; + + /** */ + private static String CACHE_ORG = "org"; + + /** */ + private static String CACHE_PERSON = "person"; + + /** */ + private static String CACHE_PERSON2 = "person-2"; + + /** */ + private static String CACHE_POSITION = "pos"; + + /** */ + private static Ignite client; + + /** */ + private static CountDownLatch latch; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration c = super.getConfiguration(gridName); + + TcpDiscoverySpi disco = new TcpDiscoverySpi(); + + disco.setIpFinder(IP_FINDER); + + c.setDiscoverySpi(disco); + + List ccfgs = new ArrayList<>(); + + ccfgs.add(buildCacheConfiguration(CACHE_ORG)); + ccfgs.add(buildCacheConfiguration(CACHE_PERSON)); + ccfgs.add(buildCacheConfiguration(CACHE_POSITION)); + + ccfgs.add(buildCacheConfiguration(CACHE_PERSON).setName(CACHE_PERSON2)); + + c.setCacheConfiguration(ccfgs.toArray(new CacheConfiguration[ccfgs.size()])); + + if (gridName.equals(NODE_CLIENT)) + c.setClientMode(true); + + return c; + } + + /** */ + private CacheConfiguration buildCacheConfiguration(String name) { + if (name.equals(CACHE_ORG)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_ORG); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + QueryEntity entity = new QueryEntity(Integer.class, Organization.class); + + ccfg.setQueryEntities(Collections.singletonList(entity)); + + ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); + + return ccfg; + } + if (name.equals(CACHE_PERSON)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_PERSON); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + QueryEntity entity = new QueryEntity(PersonKey.class, Person.class); + + ccfg.setQueryEntities(Collections.singletonList(entity)); + + ccfg.setKeyConfiguration(new CacheKeyConfiguration(PersonKey.class)); + + ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); + + return ccfg; + } + if (name.equals(CACHE_POSITION)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_POSITION); + + ccfg.setCacheMode(CacheMode.REPLICATED); + + QueryEntity entity = new QueryEntity(Integer.class, Position.class); + + ccfg.setQueryEntities(Collections.singletonList(entity)); + + ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); + + return ccfg; + } + + assert false; + + return null; + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startGrids(NODE_COUNT); + + client = startGrid(NODE_CLIENT); + + awaitPartitionMapExchange(); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + checkNoLeaks(); + + super.afterTestsStopped(); + + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + Ignite client = grid(NODE_CLIENT); + + // Stop additional node that is started in one of the test. + stopGrid(NODE_COUNT + 1); + + awaitPartitionMapExchange(); + + client.cache(CACHE_PERSON).clear(); + client.cache(CACHE_PERSON2).clear(); + client.cache(CACHE_ORG).clear(); + client.cache(CACHE_POSITION).clear(); + } + + // PARTITIONED Integer -> Organization (name, int rate, updated) + // PARTITIONED PersonKey(orgId, id) -> Person (name, int position, amount, updated) + // REPLICATED Position (id, name, int rate) + /** */ + public void testUpdate() throws Exception { + fillCaches(); + + String text = "UPDATE \"person\".Person SET amount = amount * 2 WHERE amount > 0"; + + checkUpdate(new SqlFieldsQuery(text), CACHE_PERSON, CACHE_PERSON2); + + compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); + } + + /** */ + public void testUpdateFastKey() throws Exception { + fillCaches(); + + String text = "UPDATE \"person\".Person SET amount = ? WHERE orgId = ? and id = ?"; + + checkUpdate(new SqlFieldsQuery(text).setArgs(100, 1, 1), CACHE_PERSON, CACHE_PERSON2); + + compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); + } + + /** */ + public void testUpdateLimit() throws Exception { + fillCaches(); + + // UPDATE LIMIT can't produce stable/repeatable results (and UPDATE ORDER BY is ignored by H2) + // So we must first get the count and use it later as limit + List> r = grid(NODE_CLIENT).context().query().querySqlFieldsNoCache( + new SqlFieldsQuery("SELECT COUNT(*) FROM \"person\".Person WHERE position = ?").setArgs(1), false).getAll(); + + int count = ((Number)r.get(0).get(0)).intValue(); + + String text = "UPDATE \"person\".Person SET amount = amount * 2 WHERE position = ? LIMIT ?"; + + checkUpdate(new SqlFieldsQuery(text).setArgs(1, count), CACHE_PERSON, CACHE_PERSON2); + + compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); + } + + /** */ + public void testUpdateWhereSubquery() throws Exception { + fillCaches(); + + String text = "UPDATE \"person\".Person SET amount = amount * 2 " + + "WHERE amount > 0 AND position IN (SELECT p._key FROM \"pos\".Position p WHERE rate > 3)"; + + checkUpdate(new SqlFieldsQuery(text), CACHE_PERSON, CACHE_PERSON2); + + compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); + } + + /** */ + public void testUpdateSetSubquery() throws Exception { + fillCaches(); + + String text = "UPDATE \"person\".Person p SET amount = " + + "(SELECT o.rate FROM \"pos\".Position o WHERE p.position = o._key)"; + + checkUpdate(new SqlFieldsQuery(text), CACHE_PERSON, CACHE_PERSON2); + + compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); + } + + /** */ + public void testUpdateSetTableSubquery() throws Exception { + fillCaches(); + + String text = "UPDATE \"person\".Person p SET (amount, updated) = " + + "(SELECT o.rate, CURDATE() FROM \"pos\".Position o WHERE p.position = o._key)"; + + checkUpdate(new SqlFieldsQuery(text), CACHE_PERSON, CACHE_PERSON2); + + compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); + } + + // PARTITIONED Integer -> Organization (name, int rate, updated) + // PARTITIONED PersonKey(orgId, id) -> Person (name, int position, amount, updated) + // REPLICATED Position (id, name, int rate) + // + /** */ + public void testInsertValues() throws Exception { + String sqlText = "INSERT INTO \"person\".Person () VALUES ()"; + } + + /** */ + public void testInsertFromSelect() throws Exception { + // INSERT INTO tblName (fields..) SELECT + } + + /** */ + public void testInsertFromSelectJoin() throws Exception { + // INSERT INTO tblName (fields..) SELECT JOIN + } + + /** */ + public void testInsertFromSelectOrderBy() throws Exception { + // INSERT INTO tblName (fields..) SELECT ORDER BY + } + + /** */ + public void testInsertFromSelectGroupBy() throws Exception { + // INSERT INTO tblName (fields..) SELECT GROUP BY + } + + /** */ + public void testInsertFromSelectSubquery() throws Exception { + // INSERT INTO tblName (fields..) SELECT WHERE SUB-QUERY + } + + /** */ + public void testInsertFromSelectLimitOffset() throws Exception { + // TODO: OFFSET/LIMIT can't provide stable/repeatable results + // INSERT INTO tblName (fields..) SELECT OFFSET LIMIT + } + + /** */ + public void testInsertFromSelectDistinct() throws Exception { + // INSERT INTO tblName (fields..) SELECT DISTINCT + } + + /** */ + public void testInsertFromSelectUnion() throws Exception { + // INSERT INTO tblName (fields..) SELECT UNION SELECT + } + + /** */ + public void testInsertSetField() throws Exception { + // INSERT INTO tblName (fields..) SET field = ?, field2 = ? + } + + /** */ + public void testInsertSetFieldSubquery() throws Exception { + // INSERT INTO tblName (fields..) SET field = (SUB-QUERY), field2 = (SUB-QUERY) + } + + // PARTITIONED Integer -> Organization (name, int rate, updated) + // PARTITIONED PersonKey(orgId, id) -> Person (name, int position, amount, updated) + // REPLICATED Position (id, name, int rate) + /** */ + public void testMergeValues() throws Exception { + // MERGE INTO tblName (fields..) VALUES (vals..) + } + + /** */ + public void testMergeFromSelectJoin() throws Exception { + // MERGE INTO tblName (fields..) SELECT JOIN + } + + /** */ + public void testMergeFromSelectOrderBy() throws Exception { + // MERGE INTO tblName (fields..) SELECT ORDER BY + } + + /** */ + public void testMergeFromSelectGroupBy() throws Exception { + // MERGE INTO tblName (fields..) SELECT GROUP BY + } + + /** */ + public void testMergeFromSelectDistinct() throws Exception { + // MERGE INTO tblName (fields..) SELECT DISTINCT + } + + /** */ + public void testMergeFromSelectLimitOffset() throws Exception { + // TODO: OFFSET/LIMIT can't provide stable/repeatable results + // MERGE INTO tblName (fields..) SELECT OFFSET LIMIT + } + + /** */ + public void testMergeFromSelectUnion() throws Exception { + // MERGE INTO tblName (fields..) SELECT UNION SELECT + } + + /** */ + public void testMergeFromSelectSubquery() throws Exception { + // MERGE INTO tblName (fields..) SELECT WHERE SUBQUERY + } + + // PARTITIONED Integer -> Organization (name, int rate, updated) + // PARTITIONED PersonKey(orgId, id) -> Person (name, int position, amount, updated) + // REPLICATED Position (id, name, int rate) + /** */ + public void testDelete() throws Exception { + fillCaches(); + + String text = "DELETE FROM \"person\".Person WHERE position = ?"; + + checkUpdate(new SqlFieldsQuery(text).setArgs(1), CACHE_PERSON, CACHE_PERSON2); + + compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); + } + + /** */ + public void testDeleteTop() throws Exception { + fillCaches(); + // TOP/LIMIT can't provide stable/repeatable results, so we need to get count first to use it as limit + List> r = grid(NODE_CLIENT).context().query().querySqlFieldsNoCache( + new SqlFieldsQuery("SELECT COUNT(*) FROM \"person\".Person WHERE position = ?").setArgs(1), false).getAll(); + + int count = ((Number)r.get(0).get(0)).intValue(); + + String text = "DELETE TOP ? FROM \"person\".Person WHERE position = ?"; + + checkUpdate(new SqlFieldsQuery(text).setArgs(count, 1), CACHE_PERSON, CACHE_PERSON2); + + compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); + } + + /** */ + public void testDeleteWhereSubquery() throws Exception { + fillCaches(); + + String text = "DELETE FROM \"person\".Person " + + "WHERE position IN (SELECT o._key FROM \"pos\".Position o WHERE rate > 3)"; + + checkUpdate(new SqlFieldsQuery(text), CACHE_PERSON, CACHE_PERSON2); + + compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); + } + + /** */ + public void testSimpleUpdateDistributedReplicated() throws Exception { + fillCaches(); + + IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_POSITION); + + Position p = cache.get(1); + + List> r = cache.query(new SqlFieldsQuery("UPDATE Position p SET name = CONCAT('A ', name)") + .setUpdateOnServer(true)).getAll(); + + assertEquals((long)cache.size(), r.get(0).get(0)); + + assertEquals(cache.get(1).name, "A " + p.name); + } + + /** */ + public void testSimpleUpdateDistributedPartitioned() throws Exception { + fillCaches(); + + IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); + + List> r = cache.query(new SqlFieldsQuery( + "UPDATE Person SET position = CASEWHEN(position = 1, 1, position - 1)") + .setUpdateOnServer(true)).getAll(); + + assertEquals((long)cache.size(), r.get(0).get(0)); + } + + /** */ + public void testDistributedUpdateFailedKeys() throws Exception { + // UPDATE can produce failed keys due to concurrent modification + fillCaches(); + + final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() { + return cache.query(new SqlFieldsQuery("UPDATE Organization SET rate = Modify(_key, rate - 1)") + .setUpdateOnServer(true)); + } + }, CacheException.class, "Failed to update some keys because they had been modified concurrently"); + } + + /** */ + public void testDistributedUpdateFail() throws Exception { + fillCaches(); + + final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() { + return cache.query(new SqlFieldsQuery("UPDATE Person SET name = Fail(name)") + .setUpdateOnServer(true)); + } + }, CacheException.class, "Failed to execute SQL query"); + } + + /** */ + @SuppressWarnings("ConstantConditions") + public void testQueryParallelism() throws Exception { + String cacheName = CACHE_ORG + "x4"; + + CacheConfiguration cfg = buildCacheConfiguration(CACHE_ORG) + .setQueryParallelism(4) + .setName(cacheName); + + IgniteCache cache = grid(NODE_CLIENT).createCache(cfg); + + for (int i = 0; i < 1024; i++) + cache.put(i, new Organization("Acme Inc #" + i, 0)); + + List> r = cache.query(new SqlFieldsQuery("UPDATE \"" + cacheName + + "\".Organization o SET name = UPPER(name)").setUpdateOnServer(true)).getAll(); + + assertEquals((long)cache.size(), r.get(0).get(0)); + } + + /** */ + public void testEvents() throws Exception { + final CountDownLatch latch = new CountDownLatch(NODE_COUNT); + + final IgnitePredicate pred = new IgnitePredicate() { + @Override public boolean apply(Event evt) { + assert evt instanceof CacheQueryExecutedEvent; + + CacheQueryExecutedEvent qe = (CacheQueryExecutedEvent)evt; + + assertNotNull(qe.clause()); + + latch.countDown(); + + return true; + } + }; + + for (int idx = 0; idx < NODE_COUNT; idx++) + grid(idx).events().localListen(pred, EVT_CACHE_QUERY_EXECUTED); + + IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); + + for (int i = 0; i < 1024; i++) + cache.put(i, new Organization("Acme Inc #" + i, 0)); + + cache.query(new SqlFieldsQuery("UPDATE \"org\".Organization o SET name = UPPER(name)") + .setUpdateOnServer(true)).getAll(); + + assertTrue(latch.await(5000, MILLISECONDS)); + + for (int idx = 0; idx < NODE_COUNT; idx++) + grid(idx).events().stopLocalListen(pred); + } + + /** */ + public void testSpecificPartitionsUpdate() throws Exception { + fillCaches(); + + Affinity aff = grid(NODE_CLIENT).affinity(CACHE_PERSON); + + int numParts = aff.partitions(); + int parts[] = new int[numParts / 2]; + + for (int idx = 0; idx < numParts / 2; idx++) + parts[idx] = idx * 2; + + IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); + + // UPDATE over even partitions + cache.query(new SqlFieldsQuery("UPDATE Person SET position = 0") + .setUpdateOnServer(true) + .setPartitions(parts)); + + List> rows = cache.query(new SqlFieldsQuery("SELECT _key, position FROM Person")).getAll(); + + for (List row : rows) { + PersonKey personKey = (PersonKey)row.get(0); + int pos = ((Number)row.get(1)).intValue(); + int part = aff.partition(personKey); + + assertTrue((part % 2 == 0) ^ (pos != 0)); + } + } + + /** */ + public void testCancel() throws Exception { + fillCaches(); + + final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); + + final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { + @Override public Object call() { + return cache.query(new SqlFieldsQuery("UPDATE Organization SET name = WAIT(name)") + .setUpdateOnServer(true)); + } + }); + + GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + Collection qCol = + grid(NODE_CLIENT).context().query().runningQueries(0); + + if (qCol.isEmpty()) + return false; + + for (GridRunningQueryInfo queryInfo : qCol) + queryInfo.cancel(); + + return true; + } + }, 5000); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws IgniteCheckedException { + return fut.get(); + } + }, IgniteCheckedException.class, "Future was cancelled"); + } + + /** */ + public void testNodeStopDuringUpdate() throws Exception { + startGrid(NODE_COUNT + 1); + + awaitPartitionMapExchange(); + + fillCaches(); + + latch = new CountDownLatch(NODE_COUNT + 1); + + final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); + + final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { + @Override public Object call() { + return cache.query(new SqlFieldsQuery("UPDATE Organization SET name = COUNTANDWAIT(name)") + .setUpdateOnServer(true)); + } + }); + + assertTrue(latch.await(5000, MILLISECONDS)); + + stopGrid(NODE_COUNT + 1); + + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws IgniteCheckedException { + return fut.get(); + } + }, IgniteCheckedException.class, "Update failed because map node left topology"); + } + + /** + * Ensure there are no leaks in data structures associated with distributed dml execution. + */ + private void checkNoLeaks() { + GridQueryProcessor qryProc = grid(NODE_CLIENT).context().query(); + + IgniteH2Indexing h2Idx = GridTestUtils.getFieldValue(qryProc, GridQueryProcessor.class, "idx"); + + GridReduceQueryExecutor rdcQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "rdcQryExec"); + + Map updRuns = GridTestUtils.getFieldValue(rdcQryExec, GridReduceQueryExecutor.class, "updRuns"); + + assertEquals(0, updRuns.size()); + + for (int idx = 0; idx < NODE_COUNT; idx++) { + qryProc = grid(idx).context().query(); + + h2Idx = GridTestUtils.getFieldValue(qryProc, GridQueryProcessor.class, "idx"); + + GridMapQueryExecutor mapQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec"); + + Map qryRess = GridTestUtils.getFieldValue(mapQryExec, GridMapQueryExecutor.class, "qryRess"); + + for (Object obj : qryRess.values()) { + Map updCancels = GridTestUtils.getFieldValue(obj, "updCancels"); + + assertEquals(0, updCancels.size()); + } + } + } + + /** */ + private void checkUpdate(SqlFieldsQuery updQry, String cacheNameA, String cacheNameB) { + GridQueryProcessor queryProc = grid(NODE_CLIENT).context().query(); + + List> r1 = queryProc.querySqlFieldsNoCache(new SqlFieldsQuery(updQry) + .setUpdateOnServer(true), true).getAll(); + + List> r2 = queryProc.querySqlFieldsNoCache(new SqlFieldsQuery(updQry) + .setSql(updQry.getSql().replace(cacheNameA, cacheNameB)) + .setUpdateOnServer(false), true).getAll(); + + assertNotNull(r1); + assertNotNull(r2); + + assertEquals(1, r1.size()); + assertEquals(1, r2.size()); + + assertEquals(r1.get(0).get(0), r2.get(0).get(0)); + + assertTrue(((Number)r1.get(0).get(0)).intValue() > 0); + } + + /** */ + private void compareTablesContent(String cacheNameA, String cacheNameB, String tableName) throws Exception { + IgniteEx client = grid(NODE_CLIENT); + + GridQueryProcessor queryProc = client.context().query(); + + assertEquals(client.cache(cacheNameA).size(), client.cache(cacheNameB).size()); + + int size = client.cache(cacheNameA).size(); + + String sqlText = "SELECT COUNT(*) FROM \"" + cacheNameA + "\"." + tableName + + " a JOIN \"" + cacheNameB + "\"." + tableName + " b ON a._key = b._key " + + "WHERE a._val = b._val"; + + List> r = queryProc.querySqlFieldsNoCache(new SqlFieldsQuery(sqlText), true).getAll(); + + assertNotNull(r); + + assertEquals(1, r.size()); + + assertEquals(size, ((Number)r.get(0).get(0)).intValue()); + } + + /** */ + private void checkResults(List> a, List> b) { + assertNotNull(a); + assertNotNull(b); + + assertEquals(a.size(), b.size()); + + assertTrue(a.size() > 0); + + int sz = a.size(); + for (int i = 0; i < sz; ++i) { + List ra = a.get(i); + List rb = b.get(i); + + assertNotNull(ra); + assertNotNull(rb); + + assertEquals(ra.size(), rb.size()); + + assertTrue(ra.size() > 0); + + int rSize = ra.size(); + + for (int j = 0; j < rSize; ++j) + assertEquals(ra.get(j), rb.get(j)); + } + } + + /** */ + private void fillCaches() { + Ignite client = grid(NODE_CLIENT); + + IgniteCache posCache = client.cache(CACHE_POSITION); + + // Generate positions + Position[] positions = new Position[] { + new Position(1, "High Ranking Officer", 1), + new Position(2, "Administrative worker", 3), + new Position(3, "Worker", 7), + new Position(4, "Security", 2), + new Position(5, "Cleaner", 1) + }; + + for (Position pos: positions) + posCache.put(pos.id, pos); + + // Generate organizations + String[] forms = new String[] {" Inc", " Co", " AG", " Industries"}; + String[] orgNames = new String[] {"Acme", "Sierra", "Mesa", "Umbrella", "Robotics"}; + String[] names = new String[] {"Mary", "John", "William", "Tom", "Basil", "Ann", "Peter"}; + + IgniteCache personCache = client.cache(CACHE_PERSON); + IgniteCache personCache2 = client.cache(CACHE_PERSON2); + + IgniteCache orgCache = client.cache(CACHE_ORG); + + int orgId = 0; + int personId = 0; + + for (String orgName : produceCombination(orgNames, orgNames, forms)) { + Organization org = new Organization(orgName, 1 + orgId); + + orgCache.put(++orgId, org); + + // Generate persons + + List personNames = produceCombination(names, names, new String[]{"s"}); + + int positionId = 0; + int posCounter = 0; + + for (String name : personNames) { + PersonKey pKey = new PersonKey(orgId, ++personId); + + if (positions[positionId].rate < posCounter++) { + posCounter = 0; + positionId = (positionId + 1) % positions.length; + } + + Person person = new Person(name, positions[positionId].id, org.rate * positions[positionId].rate); + + personCache.put(pKey, person); + personCache2.put(pKey, person); + } + } + } + + /** */ + private List produceCombination(String[] a, String[] b, String[] ends) { + List res = new ArrayList<>(); + + for (String s1 : a) { + for (String s2 : b) { + if (!s1.equals(s2)) { + String end = ends[ThreadLocalRandom8.current().nextInt(ends.length)]; + + res.add(s1 + " " + s2 + end); + } + } + } + + return res; + } + + /** */ + private static class Organization { + /** */ + @QuerySqlField + String name; + + @QuerySqlField + int rate; + + @QuerySqlField + Date updated; + + /** */ + public Organization() { + // No-op. + } + + /** + * @param name Organization name. + */ + public Organization(String name, int rate) { + this.name = name; + this.rate = rate; + this.updated = new Date(System.currentTimeMillis()); + } + } + + /** */ + public static class PersonKey { + /** */ + @AffinityKeyMapped + @QuerySqlField + private Integer orgId; + + /** */ + @QuerySqlField + private Integer id; + + /** */ + PersonKey(int orgId, int id) { + this.orgId = orgId; + this.id = id; + } + } + + /** */ + public static class Person { + /** */ + @QuerySqlField + String name; + + @QuerySqlField + int position; + + @QuerySqlField + int amount; + + @QuerySqlField + Date updated; + + /** */ + private Person(String name, int position, int amount) { + this.name = name; + this.position = position; + this.amount = amount; + this.updated = new Date(System.currentTimeMillis()); + } + + @Override public int hashCode() { + return (name==null? 0: name.hashCode()) ^ position ^ amount ^ (updated == null ? 0 : updated.hashCode()); + } + + @Override public boolean equals(Object obj) { + if (obj == null) + return false; + + if (!obj.getClass().equals(Person.class)) + return false; + + Person other = (Person)obj; + + return F.eq(name, other.name) && position == other.position && + amount == other.amount && F.eq(updated, other.updated); + } + + } + + /** */ + private static class Position { + /** */ + @QuerySqlField + int id; + + /** */ + @QuerySqlField + String name; + + /** */ + @QuerySqlField + int rate; + + /** */ + public Position(int id, String name, int rate) { + this.id = id; + this.name = name; + this.rate = rate; + } + } + + /** */ + @QuerySqlFunction + public static String Fail(String param) { + throw new IgniteSQLException("Fail() called"); + } + + /** */ + @QuerySqlFunction + public static String Wait(String param) { + try { + Thread.sleep(3000); + } + catch (InterruptedException ignore) { + // No-op + } + return param; + } + + /** */ + @QuerySqlFunction + public static String CountAndWait(String param) { + try { + latch.countDown(); + + Thread.sleep(3000); + } + catch (InterruptedException ignore) { + // No-op + } + return param; + } + + /** */ + @QuerySqlFunction + public static int Modify(final int id, final int rate) { + try { + GridTestUtils.runAsync(new Callable() { + @Override public Object call() { + IgniteCache cache = client.cache(CACHE_ORG); + + cache.put(id, new Organization("Acme Inc #" + id, rate + 1)); + + return null; + } + }).get(); + } + catch (Exception e) { + // No-op + } + + return rate - 1; + } +} From 88bbc3e8b0e050ae41ad92e94fc1f4e73062625c Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Sat, 7 Oct 2017 23:10:47 +0300 Subject: [PATCH 23/40] IGNITE-6024: Added tests --- .../query/h2/twostep/MapNodeResults.java | 6 +- .../IgniteSqlDistributedDmlFlagSelfTest.java | 777 ++++++++++++++++++ .../IgniteSqlDistributedDmlSelfTest.java | 287 +------ .../IgniteCacheQuerySelfTestSuite.java | 2 + 4 files changed, 817 insertions(+), 255 deletions(-) create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapNodeResults.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapNodeResults.java index 3fcbcee84dc4b..c0637eae3cac2 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapNodeResults.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapNodeResults.java @@ -125,7 +125,11 @@ public MapQueryResults put(long reqId, int segmentId, MapQueryResults qr) { * @return Cancel state. */ public GridQueryCancel putUpdate(long reqId) { - return updCancels.put(reqId, new GridQueryCancel()); + GridQueryCancel cancel = new GridQueryCancel(); + + updCancels.put(reqId, cancel); + + return cancel; } /** diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java new file mode 100644 index 0000000000000..18d34c7e3ecd6 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java @@ -0,0 +1,777 @@ +package org.apache.ignite.internal.processors.query; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.cache.Cache; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.lang.IgniteCallable; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** */ +public class IgniteSqlDistributedDmlFlagSelfTest extends GridCommonAbstractTest { + /** IP finder. */ + private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** */ + private static int NODE_COUNT = 4; + + /** */ + private static String NODE_CLIENT = "client"; + + /** */ + private static String CACHE_ACCOUNT = "acc"; + + /** */ + private static String CACHE_REPORT = "rep"; + + /** */ + private static String CACHE_STOCK = "stock"; + + /** */ + private static String CACHE_TRADE = "trade"; + + /** */ + private static String CACHE_LIST = "list"; + + /** */ + private static IgniteEx client; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration c = super.getConfiguration(gridName); + + TcpDiscoverySpi disco = new TcpDiscoverySpi(); + + disco.setIpFinder(IP_FINDER); + + c.setDiscoverySpi(disco); + + List ccfgs = new ArrayList<>(); + + ccfgs.add(buildCacheConfiguration(CACHE_ACCOUNT)); + ccfgs.add(buildCacheConfiguration(CACHE_STOCK)); + ccfgs.add(buildCacheConfiguration(CACHE_TRADE)); + + ccfgs.add(buildCacheConfiguration(CACHE_REPORT)); + ccfgs.add(buildCacheConfiguration(CACHE_LIST)); + + c.setCacheConfiguration(ccfgs.toArray(new CacheConfiguration[ccfgs.size()])); + + if (gridName.equals(NODE_CLIENT)) + c.setClientMode(true); + + return c; + } + + /** + * + * @param name + * @return + */ + private CacheConfiguration buildCacheConfiguration(String name) { + if (name.equals(CACHE_ACCOUNT)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_ACCOUNT); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + QueryEntity entity = new QueryEntity(Integer.class, Account.class); + + ccfg.setQueryEntities(Collections.singletonList(entity)); + + return ccfg; + } + if (name.equals(CACHE_STOCK)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_STOCK); + + ccfg.setCacheMode(CacheMode.REPLICATED); + + QueryEntity entity = new QueryEntity(Integer.class, Stock.class); + + ccfg.setQueryEntities(Collections.singletonList(entity)); + + return ccfg; + } + if (name.equals(CACHE_TRADE)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_TRADE); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + QueryEntity entity = new QueryEntity(Integer.class, Trade.class); + + ccfg.setQueryEntities(Collections.singletonList(entity)); + + return ccfg; + } + if (name.equals(CACHE_REPORT)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_REPORT); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + QueryEntity entity = new QueryEntity(Integer.class, Report.class); + + ccfg.setQueryEntities(Collections.singletonList(entity)); + + return ccfg; + } + if (name.equals(CACHE_LIST)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_LIST); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + QueryEntity entity = new QueryEntity(Integer.class, String.class); + + ccfg.setQueryEntities(Collections.singletonList(entity)); + + return ccfg; + } + + + assert false; + + return null; + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startGrids(NODE_COUNT); + + client = (IgniteEx)startGrid(NODE_CLIENT); + + awaitPartitionMapExchange(); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + + super.afterTestsStopped(); + + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + awaitPartitionMapExchange(); + + client.cache(CACHE_ACCOUNT).clear(); + client.cache(CACHE_STOCK).clear(); + client.cache(CACHE_TRADE).clear(); + client.cache(CACHE_REPORT).clear(); + client.cache(CACHE_LIST).clear(); + } + + /** + * + * @throws Exception If failed. + */ + public void testUpdate() throws Exception { + Map accounts = getAccounts(100, 1, 100); + + String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE depo > 0"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, new SqlFieldsQuery(text).setArgs(10), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testUpdateFastKey() throws Exception { + Map accounts = getAccounts(100, 1, 100); + + String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE _key = ?"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, + new SqlFieldsQuery(text).setArgs(10, 1), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testUpdateLimit() throws Exception { + Map accounts = getAccounts(100, 1, 100); + + String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE sn >= ? AND sn < ? LIMIT ?"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, + new SqlFieldsQuery(text).setArgs(10, 0, 10, 10), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testUpdateWhereSubquery() throws Exception { + Map accounts = getAccounts(100, 1, -100); + + Map trades = getTrades(100, 2); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "UPDATE \"trade\".Trade t SET qty = ? " + + "WHERE accountId IN (SELECT p._key FROM \"acc\".Account p WHERE depo < ?)"; + + checkUpdate(client.cache(CACHE_TRADE), trades, + new SqlFieldsQuery(text).setArgs(0, 0), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testUpdateSetSubquery() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + Map trades = getTrades(100, 2); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "UPDATE \"trade\".Trade t SET qty = " + + "(SELECT a.depo/t.price FROM \"acc\".Account a WHERE t.accountId = a._key)"; + + checkUpdate(client.cache(CACHE_TRADE), trades, + new SqlFieldsQuery(text), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testUpdateSetTableSubquery() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + Map trades = getTrades(100, 2); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "UPDATE \"trade\".Trade t SET (qty) = " + + "(SELECT a.depo/t.price FROM \"acc\".Account a WHERE t.accountId = a._key)"; + + checkUpdate(client.cache(CACHE_TRADE), trades, + new SqlFieldsQuery(text), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertValues() throws Exception { + String text = "INSERT INTO \"acc\".Account (_key, name, sn, depo)" + + " VALUES (?, ?, ?, ?), (?, ?, ?, ?)"; + + checkUpdate(client.cache(CACHE_ACCOUNT), null, + new SqlFieldsQuery(text).setArgs(1, "John Marry", 11111, 100, 2, "Marry John", 11112, 200), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertFromSelect() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "INSERT INTO \"trade\".Trade (_key, accountId, stockId, qty, price) " + + "SELECT a._key, a._key, ?, a.depo/?, ? FROM \"acc\".Account a"; + + checkUpdate(client.cache(CACHE_TRADE), null, + new SqlFieldsQuery(text).setArgs(1, 10, 10), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertFromSelectOrderBy() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "INSERT INTO \"trade\".Trade (_key, accountId, stockId, qty, price) " + + "SELECT a._key, a._key, ?, a.depo/?, ? FROM \"acc\".Account a " + + "ORDER BY a.sn DESC"; + + checkUpdate(client.cache(CACHE_TRADE), null, + new SqlFieldsQuery(text).setArgs(1, 10, 10), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertFromSelectUnion() throws Exception { + Map accounts = getAccounts(20, 1, 1000); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "INSERT INTO \"trade\".Trade (_key, accountId, stockId, qty, price) " + + "SELECT a._key, a._key, 0, a.depo, 1 FROM \"acc\".Account a " + + "UNION " + + "SELECT 101 + a2._key, a2._key, 1, a2.depo, 1 FROM \"acc\".Account a2"; + + checkUpdate(client.cache(CACHE_TRADE), null, + new SqlFieldsQuery(text), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertFromSelectGroupBy() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + Map trades = getTrades(100, 2); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + client.cache(CACHE_TRADE).putAll(trades); + + String text = "INSERT INTO \"rep\".Report (_key, accountId, spends, count) " + + "SELECT accountId, accountId, SUM(qty * price), COUNT(*) " + + "FROM \"trade\".Trade " + + "GROUP BY accountId"; + + checkUpdate(client.cache(CACHE_REPORT), null, + new SqlFieldsQuery(text), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertFromSelectDistinct() throws Exception { + Map accounts = getAccounts(100, 2, 100); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "INSERT INTO \"list\".String (_key, _val) " + + "SELECT DISTINCT sn, name FROM \"acc\".Account "; + + checkUpdate(client.cache(CACHE_LIST), null, + new SqlFieldsQuery(text), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertFromSelectJoin() throws Exception { + Map accounts = getAccounts(100, 1, 100); + Map stocks = getStocks(5); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + client.cache(CACHE_STOCK).putAll(stocks); + + String text = "INSERT INTO \"trade\".Trade(_key, accountId, stockId, qty, price) " + + "SELECT 5*a._key + s._key, a._key, s._key, ?, a.depo/? " + + "FROM \"acc\".Account a JOIN \"stock\".Stock s ON 1=1"; + + checkUpdate(client.cache(CACHE_TRADE), null, + new SqlFieldsQuery(text).setArgs(10, 10), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testDelete() throws Exception { + Map accounts = getAccounts(100, 1, 100); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "DELETE FROM \"acc\".Account WHERE sn > ?"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, + new SqlFieldsQuery(text).setArgs(10), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testDeleteTop() throws Exception { + Map accounts = getAccounts(100, 1, 100); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "DELETE TOP ? FROM \"acc\".Account WHERE sn < ?"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, + new SqlFieldsQuery(text).setArgs(10, 10), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testDeleteWhereSubquery() throws Exception { + Map accounts = getAccounts(20, 1, 100); + Map trades = getTrades(10, 2); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + client.cache(CACHE_TRADE).putAll(trades); + + String text = "DELETE FROM \"acc\".Account " + + "WHERE _key IN (SELECT t.accountId FROM \"trade\".Trade t)"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, + new SqlFieldsQuery(text), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testMergeValues() throws Exception { + Map accounts = getAccounts(1, 1, 100); + + String text = "MERGE INTO \"acc\".Account (_key, name, sn, depo)" + + " VALUES (?, ?, ?, ?), (?, ?, ?, ?)"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, + new SqlFieldsQuery(text).setArgs(0, "John Marry", 11111, 100, 1, "Marry John", 11112, 200), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testMergeFromSelectJoin() throws Exception { + Map accounts = getAccounts(100, 1, 100); + Map stocks = getStocks(5); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + client.cache(CACHE_STOCK).putAll(stocks); + + Map trades = new HashMap<>(); + + trades.put(5, new Trade(1, 1, 1, 1)); + + String text = "MERGE INTO \"trade\".Trade(_key, accountId, stockId, qty, price) " + + "SELECT 5*a._key + s._key, a._key, s._key, ?, a.depo/? " + + "FROM \"acc\".Account a JOIN \"stock\".Stock s ON 1=1"; + + checkUpdate(client.cache(CACHE_TRADE), trades, + new SqlFieldsQuery(text).setArgs(10, 10), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testMergeFromSelectOrderBy() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + Map trades = new HashMap<>(); + + trades.put(5, new Trade(1, 1, 1, 1)); + + String text = "MERGE INTO \"trade\".Trade (_key, accountId, stockId, qty, price) " + + "SELECT a._key, a._key, ?, a.depo/?, ? FROM \"acc\".Account a " + + "ORDER BY a.sn DESC"; + + checkUpdate(client.cache(CACHE_TRADE), trades, + new SqlFieldsQuery(text).setArgs(1, 10, 10), null); + } + + /** + * + * @throws Exception If failed. + */ + public void testMergeFromSelectGroupBy() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + Map trades = getTrades(100, 2); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + client.cache(CACHE_TRADE).putAll(trades); + + Map reports = new HashMap<>(); + + reports.put(5, new Report(5, 1, 1)); + + String text = "MERGE INTO \"rep\".Report (_key, accountId, spends, count) " + + "SELECT accountId, accountId, SUM(qty * price), COUNT(*) " + + "FROM \"trade\".Trade " + + "GROUP BY accountId"; + + checkUpdate(client.cache(CACHE_REPORT), reports, + new SqlFieldsQuery(text), null); + } + + /** + * + * @param num + * @param numCopy + * @param depo + * @return + */ + private Map getAccounts(int num, int numCopy, int depo) { + Map res = new HashMap<>(); + + int count = 0; + + for (int i = 0; i < num; ++i) { + String name = "John doe #" + i; + + for (int j = 0; j < numCopy; ++j) + res.put(count++, new Account(name, i, depo)); + } + + return res; + } + + /** + * + * @param num + * @return + */ + private Map getStocks(int num) { + Map res = new HashMap<>(); + + for (int i = 0; i < num; ++i) + res.put(i, new Stock("T" + i, "Stock #" + i)); + + return res; + } + + /** + * + * @param numAccounts + * @param numStocks + * @return + */ + private Map getTrades(int numAccounts, int numStocks) { + Map res = new HashMap<>(); + + int count = 0; + + for (int i = 0; i < numAccounts; ++i) { + for (int j = 0; j < numStocks; ++j) { + res.put(count++, new Trade(i, j, 100, 100)); + } + } + + return res; + } + + /** + * + * @param cache + * @param initial + * @param qry + * @param clo + * @param + * @param + */ + private void checkUpdate(IgniteCache cache, Map initial, SqlFieldsQuery qry, + IgniteCallable> clo) { + cache.clear(); + + if (!F.isEmpty(initial)) + cache.putAll(initial); + + List> updRes = cache.query(qry.setUpdateOnServer(true)).getAll(); + + Map result = new HashMap<>(cache.size()); + + for (Cache.Entry e : cache) + result.put(e.getKey(), e.getValue()); + + cache.clear(); + + if (!F.isEmpty(initial)) + cache.putAll(initial); + + List> updRes2 = cache.query(qry.setUpdateOnServer(false)).getAll(); + + assertTrue(((Number)updRes.get(0).get(0)).intValue() > 0); + + assertEquals(((Number)updRes.get(0).get(0)).intValue(), ((Number)updRes2.get(0).get(0)).intValue()); + + assertEquals(result.size(), cache.size()); + + for (Cache.Entry e : cache) + assertEquals(e.getValue(), result.get(e.getKey())); + } + + /** */ + public class Account { + /** */ + @QuerySqlField + String name; + + /** */ + @QuerySqlField + int sn; + + /** */ + @QuerySqlField + int depo; + + /** + * + * @param name + * @param sn + * @param depo + */ + public Account(String name, int sn, int depo) { + this.name = name; + this.sn = sn; + this.depo = depo; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return (name == null ? 0 : name.hashCode()) ^ sn ^ depo; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (obj == null) + return false; + + if (!obj.getClass().equals(Account.class)) + return false; + + Account other = (Account)obj; + + return F.eq(name, other.name) && sn == other.sn && depo == other.depo; + } + } + + /** */ + public class Stock { + /** */ + @QuerySqlField + String ticker; + + /** */ + @QuerySqlField + String name; + + /** + * + * @param ticker + * @param name + */ + public Stock(String ticker, String name) { + this.ticker = ticker; + this.name = name; + } + } + + /** */ + public class Trade { + /** */ + @QuerySqlField + int accountId; + + /** */ + @QuerySqlField + int stockId; + + /** */ + @QuerySqlField + int qty; + + /** */ + @QuerySqlField + int price; + + /** + * + * @param accountId + * @param stockId + * @param qty + * @param price + */ + public Trade(int accountId, int stockId, int qty, int price) { + this.accountId = accountId; + this.stockId = stockId; + this.qty = qty; + this.price = price; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return accountId ^ stockId ^ qty ^ price; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (obj == null) + return false; + + if (!obj.getClass().equals(Trade.class)) + return false; + + Trade other = (Trade)obj; + + return accountId == other.accountId && stockId == other.stockId && + qty == other.qty && price == other.price; + } + + } + + /** */ + public class Report { + /** */ + @QuerySqlField + int accountId; + + /** */ + @QuerySqlField + int spends; + + /** */ + @QuerySqlField + int count; + + /** + * + * @param accountId + * @param spends + * @param count + */ + public Report(int accountId, int spends, int count) { + this.accountId = accountId; + this.spends = spends; + this.count = count; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return accountId ^ spends ^ count; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (obj == null) + return false; + + if (!obj.getClass().equals(Report.class)) + return false; + + Report other = (Report)obj; + + return accountId == other.accountId && spends == other.spends && + count == other.count; + } + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java index dba0283d76d90..26eece1a0c2c1 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java @@ -17,10 +17,10 @@ package org.apache.ignite.internal.processors.query; -import java.sql.Date; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; @@ -76,9 +76,6 @@ public class IgniteSqlDistributedDmlSelfTest extends GridCommonAbstractTest { /** */ private static String CACHE_PERSON = "person"; - /** */ - private static String CACHE_PERSON2 = "person-2"; - /** */ private static String CACHE_POSITION = "pos"; @@ -104,10 +101,10 @@ public class IgniteSqlDistributedDmlSelfTest extends GridCommonAbstractTest { ccfgs.add(buildCacheConfiguration(CACHE_PERSON)); ccfgs.add(buildCacheConfiguration(CACHE_POSITION)); - ccfgs.add(buildCacheConfiguration(CACHE_PERSON).setName(CACHE_PERSON2)); - c.setCacheConfiguration(ccfgs.toArray(new CacheConfiguration[ccfgs.size()])); + c.setLongQueryWarningTimeout(10000); + if (gridName.equals(NODE_CLIENT)) c.setClientMode(true); @@ -185,7 +182,7 @@ private CacheConfiguration buildCacheConfiguration(String name) { /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { - Ignite client = grid(NODE_CLIENT); + super.afterTest(); // Stop additional node that is started in one of the test. stopGrid(NODE_COUNT + 1); @@ -193,236 +190,10 @@ private CacheConfiguration buildCacheConfiguration(String name) { awaitPartitionMapExchange(); client.cache(CACHE_PERSON).clear(); - client.cache(CACHE_PERSON2).clear(); client.cache(CACHE_ORG).clear(); client.cache(CACHE_POSITION).clear(); } - // PARTITIONED Integer -> Organization (name, int rate, updated) - // PARTITIONED PersonKey(orgId, id) -> Person (name, int position, amount, updated) - // REPLICATED Position (id, name, int rate) - /** */ - public void testUpdate() throws Exception { - fillCaches(); - - String text = "UPDATE \"person\".Person SET amount = amount * 2 WHERE amount > 0"; - - checkUpdate(new SqlFieldsQuery(text), CACHE_PERSON, CACHE_PERSON2); - - compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); - } - - /** */ - public void testUpdateFastKey() throws Exception { - fillCaches(); - - String text = "UPDATE \"person\".Person SET amount = ? WHERE orgId = ? and id = ?"; - - checkUpdate(new SqlFieldsQuery(text).setArgs(100, 1, 1), CACHE_PERSON, CACHE_PERSON2); - - compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); - } - - /** */ - public void testUpdateLimit() throws Exception { - fillCaches(); - - // UPDATE LIMIT can't produce stable/repeatable results (and UPDATE ORDER BY is ignored by H2) - // So we must first get the count and use it later as limit - List> r = grid(NODE_CLIENT).context().query().querySqlFieldsNoCache( - new SqlFieldsQuery("SELECT COUNT(*) FROM \"person\".Person WHERE position = ?").setArgs(1), false).getAll(); - - int count = ((Number)r.get(0).get(0)).intValue(); - - String text = "UPDATE \"person\".Person SET amount = amount * 2 WHERE position = ? LIMIT ?"; - - checkUpdate(new SqlFieldsQuery(text).setArgs(1, count), CACHE_PERSON, CACHE_PERSON2); - - compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); - } - - /** */ - public void testUpdateWhereSubquery() throws Exception { - fillCaches(); - - String text = "UPDATE \"person\".Person SET amount = amount * 2 " + - "WHERE amount > 0 AND position IN (SELECT p._key FROM \"pos\".Position p WHERE rate > 3)"; - - checkUpdate(new SqlFieldsQuery(text), CACHE_PERSON, CACHE_PERSON2); - - compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); - } - - /** */ - public void testUpdateSetSubquery() throws Exception { - fillCaches(); - - String text = "UPDATE \"person\".Person p SET amount = " + - "(SELECT o.rate FROM \"pos\".Position o WHERE p.position = o._key)"; - - checkUpdate(new SqlFieldsQuery(text), CACHE_PERSON, CACHE_PERSON2); - - compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); - } - - /** */ - public void testUpdateSetTableSubquery() throws Exception { - fillCaches(); - - String text = "UPDATE \"person\".Person p SET (amount, updated) = " + - "(SELECT o.rate, CURDATE() FROM \"pos\".Position o WHERE p.position = o._key)"; - - checkUpdate(new SqlFieldsQuery(text), CACHE_PERSON, CACHE_PERSON2); - - compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); - } - - // PARTITIONED Integer -> Organization (name, int rate, updated) - // PARTITIONED PersonKey(orgId, id) -> Person (name, int position, amount, updated) - // REPLICATED Position (id, name, int rate) - // - /** */ - public void testInsertValues() throws Exception { - String sqlText = "INSERT INTO \"person\".Person () VALUES ()"; - } - - /** */ - public void testInsertFromSelect() throws Exception { - // INSERT INTO tblName (fields..) SELECT - } - - /** */ - public void testInsertFromSelectJoin() throws Exception { - // INSERT INTO tblName (fields..) SELECT JOIN - } - - /** */ - public void testInsertFromSelectOrderBy() throws Exception { - // INSERT INTO tblName (fields..) SELECT ORDER BY - } - - /** */ - public void testInsertFromSelectGroupBy() throws Exception { - // INSERT INTO tblName (fields..) SELECT GROUP BY - } - - /** */ - public void testInsertFromSelectSubquery() throws Exception { - // INSERT INTO tblName (fields..) SELECT WHERE SUB-QUERY - } - - /** */ - public void testInsertFromSelectLimitOffset() throws Exception { - // TODO: OFFSET/LIMIT can't provide stable/repeatable results - // INSERT INTO tblName (fields..) SELECT OFFSET LIMIT - } - - /** */ - public void testInsertFromSelectDistinct() throws Exception { - // INSERT INTO tblName (fields..) SELECT DISTINCT - } - - /** */ - public void testInsertFromSelectUnion() throws Exception { - // INSERT INTO tblName (fields..) SELECT UNION SELECT - } - - /** */ - public void testInsertSetField() throws Exception { - // INSERT INTO tblName (fields..) SET field = ?, field2 = ? - } - - /** */ - public void testInsertSetFieldSubquery() throws Exception { - // INSERT INTO tblName (fields..) SET field = (SUB-QUERY), field2 = (SUB-QUERY) - } - - // PARTITIONED Integer -> Organization (name, int rate, updated) - // PARTITIONED PersonKey(orgId, id) -> Person (name, int position, amount, updated) - // REPLICATED Position (id, name, int rate) - /** */ - public void testMergeValues() throws Exception { - // MERGE INTO tblName (fields..) VALUES (vals..) - } - - /** */ - public void testMergeFromSelectJoin() throws Exception { - // MERGE INTO tblName (fields..) SELECT JOIN - } - - /** */ - public void testMergeFromSelectOrderBy() throws Exception { - // MERGE INTO tblName (fields..) SELECT ORDER BY - } - - /** */ - public void testMergeFromSelectGroupBy() throws Exception { - // MERGE INTO tblName (fields..) SELECT GROUP BY - } - - /** */ - public void testMergeFromSelectDistinct() throws Exception { - // MERGE INTO tblName (fields..) SELECT DISTINCT - } - - /** */ - public void testMergeFromSelectLimitOffset() throws Exception { - // TODO: OFFSET/LIMIT can't provide stable/repeatable results - // MERGE INTO tblName (fields..) SELECT OFFSET LIMIT - } - - /** */ - public void testMergeFromSelectUnion() throws Exception { - // MERGE INTO tblName (fields..) SELECT UNION SELECT - } - - /** */ - public void testMergeFromSelectSubquery() throws Exception { - // MERGE INTO tblName (fields..) SELECT WHERE SUBQUERY - } - - // PARTITIONED Integer -> Organization (name, int rate, updated) - // PARTITIONED PersonKey(orgId, id) -> Person (name, int position, amount, updated) - // REPLICATED Position (id, name, int rate) - /** */ - public void testDelete() throws Exception { - fillCaches(); - - String text = "DELETE FROM \"person\".Person WHERE position = ?"; - - checkUpdate(new SqlFieldsQuery(text).setArgs(1), CACHE_PERSON, CACHE_PERSON2); - - compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); - } - - /** */ - public void testDeleteTop() throws Exception { - fillCaches(); - // TOP/LIMIT can't provide stable/repeatable results, so we need to get count first to use it as limit - List> r = grid(NODE_CLIENT).context().query().querySqlFieldsNoCache( - new SqlFieldsQuery("SELECT COUNT(*) FROM \"person\".Person WHERE position = ?").setArgs(1), false).getAll(); - - int count = ((Number)r.get(0).get(0)).intValue(); - - String text = "DELETE TOP ? FROM \"person\".Person WHERE position = ?"; - - checkUpdate(new SqlFieldsQuery(text).setArgs(count, 1), CACHE_PERSON, CACHE_PERSON2); - - compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); - } - - /** */ - public void testDeleteWhereSubquery() throws Exception { - fillCaches(); - - String text = "DELETE FROM \"person\".Person " + - "WHERE position IN (SELECT o._key FROM \"pos\".Position o WHERE rate > 3)"; - - checkUpdate(new SqlFieldsQuery(text), CACHE_PERSON, CACHE_PERSON2); - - compareTablesContent(CACHE_PERSON, CACHE_PERSON2, "Person"); - } - /** */ public void testSimpleUpdateDistributedReplicated() throws Exception { fillCaches(); @@ -568,6 +339,8 @@ public void testSpecificPartitionsUpdate() throws Exception { /** */ public void testCancel() throws Exception { + latch = new CountDownLatch(NODE_COUNT + 1); + fillCaches(); final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); @@ -594,6 +367,8 @@ public void testCancel() throws Exception { } }, 5000); + latch.await(5000, MILLISECONDS); + GridTestUtils.assertThrows(log, new Callable() { @Override public Object call() throws IgniteCheckedException { return fut.get(); @@ -609,18 +384,26 @@ public void testNodeStopDuringUpdate() throws Exception { fillCaches(); - latch = new CountDownLatch(NODE_COUNT + 1); + latch = new CountDownLatch(NODE_COUNT + 1 + 1); final IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_ORG); final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { @Override public Object call() { - return cache.query(new SqlFieldsQuery("UPDATE Organization SET name = COUNTANDWAIT(name)") + return cache.query(new SqlFieldsQuery("UPDATE Organization SET name = WAIT(name)") .setUpdateOnServer(true)); } }); - assertTrue(latch.await(5000, MILLISECONDS)); + final CountDownLatch finalLatch = latch; + + assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() { + @Override public boolean apply() { + return finalLatch.getCount() == 1; + } + }, 5000)); + + latch.countDown(); stopGrid(NODE_COUNT + 1); @@ -759,7 +542,6 @@ private void fillCaches() { String[] names = new String[] {"Mary", "John", "William", "Tom", "Basil", "Ann", "Peter"}; IgniteCache personCache = client.cache(CACHE_PERSON); - IgniteCache personCache2 = client.cache(CACHE_PERSON2); IgniteCache orgCache = client.cache(CACHE_ORG); @@ -789,7 +571,6 @@ private void fillCaches() { Person person = new Person(name, positions[positionId].id, org.rate * positions[positionId].rate); personCache.put(pKey, person); - personCache2.put(pKey, person); } } } @@ -817,9 +598,11 @@ private static class Organization { @QuerySqlField String name; + /** */ @QuerySqlField int rate; + /** */ @QuerySqlField Date updated; @@ -862,12 +645,14 @@ public static class Person { @QuerySqlField String name; + /** */ @QuerySqlField int position; + /** */ @QuerySqlField int amount; - + /** */ @QuerySqlField Date updated; @@ -876,13 +661,16 @@ private Person(String name, int position, int amount) { this.name = name; this.position = position; this.amount = amount; + this.updated = new Date(System.currentTimeMillis()); } + /** */ @Override public int hashCode() { return (name==null? 0: name.hashCode()) ^ position ^ amount ^ (updated == null ? 0 : updated.hashCode()); } + /** */ @Override public boolean equals(Object obj) { if (obj == null) return false; @@ -895,7 +683,6 @@ private Person(String name, int position, int amount) { return F.eq(name, other.name) && position == other.position && amount == other.amount && F.eq(updated, other.updated); } - } /** */ @@ -930,21 +717,13 @@ public static String Fail(String param) { @QuerySqlFunction public static String Wait(String param) { try { - Thread.sleep(3000); - } - catch (InterruptedException ignore) { - // No-op - } - return param; - } - - /** */ - @QuerySqlFunction - public static String CountAndWait(String param) { - try { - latch.countDown(); + if (latch.getCount() > 0) { + latch.countDown(); - Thread.sleep(3000); + latch.await(5000, MILLISECONDS); + } + else + Thread.sleep(100); } catch (InterruptedException ignore) { // No-op diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java index 375d908e2b6eb..d6083cf5fc666 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java @@ -123,6 +123,7 @@ import org.apache.ignite.internal.processors.cache.query.IndexingSpiQuerySelfTest; import org.apache.ignite.internal.processors.cache.query.IndexingSpiQueryTxSelfTest; import org.apache.ignite.internal.processors.client.ClientConnectorConfigurationValidationSelfTest; +import org.apache.ignite.internal.processors.query.IgniteSqlDistributedDmlFlagSelfTest; import org.apache.ignite.internal.processors.query.IgniteSqlParameterizedQueryTest; import org.apache.ignite.internal.processors.query.h2.IgniteSqlBigIntegerKeyTest; import org.apache.ignite.internal.processors.query.IgniteQueryDedicatedPoolTest; @@ -245,6 +246,7 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgniteCacheUpdateSqlQuerySelfTest.class); suite.addTestSuite(IgniteCacheDeleteSqlQuerySelfTest.class); suite.addTestSuite(IgniteSqlDistributedDmlSelfTest.class); + suite.addTestSuite(IgniteSqlDistributedDmlFlagSelfTest.class); suite.addTestSuite(IgniteBinaryObjectQueryArgumentsTest.class); suite.addTestSuite(IgniteBinaryObjectLocalQueryArgumentsTest.class); From 35e96910bb3e99ae38e501b3b49ee1ae87e441f9 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Mon, 9 Oct 2017 09:49:49 +0300 Subject: [PATCH 24/40] IGNITE-6024: clean up --- .../IgniteSqlDistributedDmlFlagSelfTest.java | 122 +++++------ .../IgniteSqlDistributedDmlSelfTest.java | 195 +++++++++--------- 2 files changed, 159 insertions(+), 158 deletions(-) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java index 18d34c7e3ecd6..3db44e83e7220 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java @@ -15,12 +15,11 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.lang.IgniteCallable; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -/** */ +/** Tests for {@link SqlFieldsQuery#updateOnServer} flag. */ public class IgniteSqlDistributedDmlFlagSelfTest extends GridCommonAbstractTest { /** IP finder. */ private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); @@ -64,7 +63,6 @@ public class IgniteSqlDistributedDmlFlagSelfTest extends GridCommonAbstractTest ccfgs.add(buildCacheConfiguration(CACHE_ACCOUNT)); ccfgs.add(buildCacheConfiguration(CACHE_STOCK)); ccfgs.add(buildCacheConfiguration(CACHE_TRADE)); - ccfgs.add(buildCacheConfiguration(CACHE_REPORT)); ccfgs.add(buildCacheConfiguration(CACHE_LIST)); @@ -77,9 +75,10 @@ public class IgniteSqlDistributedDmlFlagSelfTest extends GridCommonAbstractTest } /** + * Creates a cache configuration. * - * @param name - * @return + * @param name Name of the cache. + * @return Cache configuration. */ private CacheConfiguration buildCacheConfiguration(String name) { if (name.equals(CACHE_ACCOUNT)) { @@ -138,7 +137,6 @@ private CacheConfiguration buildCacheConfiguration(String name) { return ccfg; } - assert false; return null; @@ -185,7 +183,7 @@ public void testUpdate() throws Exception { String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE depo > 0"; - checkUpdate(client.cache(CACHE_ACCOUNT), accounts, new SqlFieldsQuery(text).setArgs(10), null); + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, new SqlFieldsQuery(text).setArgs(10)); } /** @@ -198,7 +196,7 @@ public void testUpdateFastKey() throws Exception { String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE _key = ?"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQuery(text).setArgs(10, 1), null); + new SqlFieldsQuery(text).setArgs(10, 1)); } /** @@ -211,7 +209,7 @@ public void testUpdateLimit() throws Exception { String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE sn >= ? AND sn < ? LIMIT ?"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQuery(text).setArgs(10, 0, 10, 10), null); + new SqlFieldsQuery(text).setArgs(10, 0, 10, 10)); } /** @@ -229,7 +227,7 @@ public void testUpdateWhereSubquery() throws Exception { "WHERE accountId IN (SELECT p._key FROM \"acc\".Account p WHERE depo < ?)"; checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQuery(text).setArgs(0, 0), null); + new SqlFieldsQuery(text).setArgs(0, 0)); } /** @@ -246,7 +244,7 @@ public void testUpdateSetSubquery() throws Exception { "(SELECT a.depo/t.price FROM \"acc\".Account a WHERE t.accountId = a._key)"; checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQuery(text), null); + new SqlFieldsQuery(text)); } /** @@ -263,7 +261,7 @@ public void testUpdateSetTableSubquery() throws Exception { "(SELECT a.depo/t.price FROM \"acc\".Account a WHERE t.accountId = a._key)"; checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQuery(text), null); + new SqlFieldsQuery(text)); } /** @@ -275,7 +273,7 @@ public void testInsertValues() throws Exception { " VALUES (?, ?, ?, ?), (?, ?, ?, ?)"; checkUpdate(client.cache(CACHE_ACCOUNT), null, - new SqlFieldsQuery(text).setArgs(1, "John Marry", 11111, 100, 2, "Marry John", 11112, 200), null); + new SqlFieldsQuery(text).setArgs(1, "John Marry", 11111, 100, 2, "Marry John", 11112, 200)); } /** @@ -291,7 +289,7 @@ public void testInsertFromSelect() throws Exception { "SELECT a._key, a._key, ?, a.depo/?, ? FROM \"acc\".Account a"; checkUpdate(client.cache(CACHE_TRADE), null, - new SqlFieldsQuery(text).setArgs(1, 10, 10), null); + new SqlFieldsQuery(text).setArgs(1, 10, 10)); } /** @@ -308,7 +306,7 @@ public void testInsertFromSelectOrderBy() throws Exception { "ORDER BY a.sn DESC"; checkUpdate(client.cache(CACHE_TRADE), null, - new SqlFieldsQuery(text).setArgs(1, 10, 10), null); + new SqlFieldsQuery(text).setArgs(1, 10, 10)); } /** @@ -326,7 +324,7 @@ public void testInsertFromSelectUnion() throws Exception { "SELECT 101 + a2._key, a2._key, 1, a2.depo, 1 FROM \"acc\".Account a2"; checkUpdate(client.cache(CACHE_TRADE), null, - new SqlFieldsQuery(text), null); + new SqlFieldsQuery(text)); } /** @@ -346,7 +344,7 @@ public void testInsertFromSelectGroupBy() throws Exception { "GROUP BY accountId"; checkUpdate(client.cache(CACHE_REPORT), null, - new SqlFieldsQuery(text), null); + new SqlFieldsQuery(text)); } /** @@ -362,7 +360,7 @@ public void testInsertFromSelectDistinct() throws Exception { "SELECT DISTINCT sn, name FROM \"acc\".Account "; checkUpdate(client.cache(CACHE_LIST), null, - new SqlFieldsQuery(text), null); + new SqlFieldsQuery(text)); } /** @@ -381,7 +379,7 @@ public void testInsertFromSelectJoin() throws Exception { "FROM \"acc\".Account a JOIN \"stock\".Stock s ON 1=1"; checkUpdate(client.cache(CACHE_TRADE), null, - new SqlFieldsQuery(text).setArgs(10, 10), null); + new SqlFieldsQuery(text).setArgs(10, 10)); } /** @@ -396,7 +394,7 @@ public void testDelete() throws Exception { String text = "DELETE FROM \"acc\".Account WHERE sn > ?"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQuery(text).setArgs(10), null); + new SqlFieldsQuery(text).setArgs(10)); } /** @@ -411,7 +409,7 @@ public void testDeleteTop() throws Exception { String text = "DELETE TOP ? FROM \"acc\".Account WHERE sn < ?"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQuery(text).setArgs(10, 10), null); + new SqlFieldsQuery(text).setArgs(10, 10)); } /** @@ -429,7 +427,7 @@ public void testDeleteWhereSubquery() throws Exception { "WHERE _key IN (SELECT t.accountId FROM \"trade\".Trade t)"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQuery(text), null); + new SqlFieldsQuery(text)); } /** @@ -443,7 +441,7 @@ public void testMergeValues() throws Exception { " VALUES (?, ?, ?, ?), (?, ?, ?, ?)"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQuery(text).setArgs(0, "John Marry", 11111, 100, 1, "Marry John", 11112, 200), null); + new SqlFieldsQuery(text).setArgs(0, "John Marry", 11111, 100, 1, "Marry John", 11112, 200)); } /** @@ -466,7 +464,7 @@ public void testMergeFromSelectJoin() throws Exception { "FROM \"acc\".Account a JOIN \"stock\".Stock s ON 1=1"; checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQuery(text).setArgs(10, 10), null); + new SqlFieldsQuery(text).setArgs(10, 10)); } /** @@ -487,7 +485,7 @@ public void testMergeFromSelectOrderBy() throws Exception { "ORDER BY a.sn DESC"; checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQuery(text).setArgs(1, 10, 10), null); + new SqlFieldsQuery(text).setArgs(1, 10, 10)); } /** @@ -511,15 +509,16 @@ public void testMergeFromSelectGroupBy() throws Exception { "GROUP BY accountId"; checkUpdate(client.cache(CACHE_REPORT), reports, - new SqlFieldsQuery(text), null); + new SqlFieldsQuery(text)); } /** + * Constructs multiple Account objects. * - * @param num - * @param numCopy - * @param depo - * @return + * @param num Number of accounts. + * @param numCopy Number of copies. + * @param depo Deposit amount. + * @return Map of accounts. */ private Map getAccounts(int num, int numCopy, int depo) { Map res = new HashMap<>(); @@ -537,9 +536,10 @@ private Map getAccounts(int num, int numCopy, int depo) { } /** + * Constructs multiple Stock objects. * - * @param num - * @return + * @param num Number of stocks. + * @return Map of Stock objects. */ private Map getStocks(int num) { Map res = new HashMap<>(); @@ -551,10 +551,11 @@ private Map getStocks(int num) { } /** + * Constructs multiple Trade objects. * - * @param numAccounts - * @param numStocks - * @return + * @param numAccounts Number of accounts. + * @param numStocks Number of stocks. + * @return Map of Trade objects. */ private Map getTrades(int numAccounts, int numStocks) { Map res = new HashMap<>(); @@ -571,16 +572,15 @@ private Map getTrades(int numAccounts, int numStocks) { } /** + * Executes provided sql update with updateOnServer flag on and off and checks results are the same. * - * @param cache - * @param initial - * @param qry - * @param clo - * @param - * @param + * @param cache Cache. + * @param initial Initial content of the cache. + * @param qry Query to execute. + * @param Key type. + * @param Value type. */ - private void checkUpdate(IgniteCache cache, Map initial, SqlFieldsQuery qry, - IgniteCallable> clo) { + private void checkUpdate(IgniteCache cache, Map initial, SqlFieldsQuery qry) { cache.clear(); if (!F.isEmpty(initial)) @@ -625,12 +625,13 @@ public class Account { int depo; /** + * Constructor. * - * @param name - * @param sn - * @param depo + * @param name Name. + * @param sn ID. + * @param depo Deposit amount. */ - public Account(String name, int sn, int depo) { + Account(String name, int sn, int depo) { this.name = name; this.sn = sn; this.depo = depo; @@ -666,11 +667,12 @@ public class Stock { String name; /** + * Constructor. * - * @param ticker - * @param name + * @param ticker Short name. + * @param name Name. */ - public Stock(String ticker, String name) { + Stock(String ticker, String name) { this.ticker = ticker; this.name = name; } @@ -695,13 +697,14 @@ public class Trade { int price; /** + * Constructor. * - * @param accountId - * @param stockId - * @param qty - * @param price + * @param accountId Account id. + * @param stockId Stock id. + * @param qty Quantity. + * @param price Price. */ - public Trade(int accountId, int stockId, int qty, int price) { + Trade(int accountId, int stockId, int qty, int price) { this.accountId = accountId; this.stockId = stockId; this.qty = qty; @@ -744,12 +747,13 @@ public class Report { int count; /** + * Constructor. * - * @param accountId - * @param spends - * @param count + * @param accountId Account id. + * @param spends Spends. + * @param count Count. */ - public Report(int accountId, int spends, int count) { + Report(int accountId, int spends, int count) { this.accountId = accountId; this.spends = spends; this.count = count; diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java index 26eece1a0c2c1..e973ce2833e1f 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java @@ -41,7 +41,6 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.events.CacheQueryExecutedEvent; import org.apache.ignite.events.Event; -import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor; @@ -111,7 +110,12 @@ public class IgniteSqlDistributedDmlSelfTest extends GridCommonAbstractTest { return c; } - /** */ + /** + * Creates cache configuration. + * + * @param name Cache name. + * @return Cache configuration. + */ private CacheConfiguration buildCacheConfiguration(String name) { if (name.equals(CACHE_ORG)) { CacheConfiguration ccfg = new CacheConfiguration(CACHE_ORG); @@ -194,7 +198,10 @@ private CacheConfiguration buildCacheConfiguration(String name) { client.cache(CACHE_POSITION).clear(); } - /** */ + /** + * + * @throws Exception if failed. + */ public void testSimpleUpdateDistributedReplicated() throws Exception { fillCaches(); @@ -210,7 +217,10 @@ public void testSimpleUpdateDistributedReplicated() throws Exception { assertEquals(cache.get(1).name, "A " + p.name); } - /** */ + /** + * + * @throws Exception if failed. + */ public void testSimpleUpdateDistributedPartitioned() throws Exception { fillCaches(); @@ -223,7 +233,10 @@ public void testSimpleUpdateDistributedPartitioned() throws Exception { assertEquals((long)cache.size(), r.get(0).get(0)); } - /** */ + /** + * + * @throws Exception if failed. + */ public void testDistributedUpdateFailedKeys() throws Exception { // UPDATE can produce failed keys due to concurrent modification fillCaches(); @@ -238,7 +251,10 @@ public void testDistributedUpdateFailedKeys() throws Exception { }, CacheException.class, "Failed to update some keys because they had been modified concurrently"); } - /** */ + /** + * + * @throws Exception if failed. + */ public void testDistributedUpdateFail() throws Exception { fillCaches(); @@ -252,7 +268,10 @@ public void testDistributedUpdateFail() throws Exception { }, CacheException.class, "Failed to execute SQL query"); } - /** */ + /** + * + * @throws Exception if failed. + */ @SuppressWarnings("ConstantConditions") public void testQueryParallelism() throws Exception { String cacheName = CACHE_ORG + "x4"; @@ -272,7 +291,10 @@ public void testQueryParallelism() throws Exception { assertEquals((long)cache.size(), r.get(0).get(0)); } - /** */ + /** + * + * @throws Exception if failed. + */ public void testEvents() throws Exception { final CountDownLatch latch = new CountDownLatch(NODE_COUNT); @@ -307,7 +329,10 @@ public void testEvents() throws Exception { grid(idx).events().stopLocalListen(pred); } - /** */ + /** + * + * @throws Exception if failed. + */ public void testSpecificPartitionsUpdate() throws Exception { fillCaches(); @@ -337,7 +362,10 @@ public void testSpecificPartitionsUpdate() throws Exception { } } - /** */ + /** + * + * @throws Exception if failed. + */ public void testCancel() throws Exception { latch = new CountDownLatch(NODE_COUNT + 1); @@ -376,7 +404,10 @@ public void testCancel() throws Exception { }, IgniteCheckedException.class, "Future was cancelled"); } - /** */ + /** + * + * @throws Exception if failed. + */ public void testNodeStopDuringUpdate() throws Exception { startGrid(NODE_COUNT + 1); @@ -445,80 +476,9 @@ private void checkNoLeaks() { } } - /** */ - private void checkUpdate(SqlFieldsQuery updQry, String cacheNameA, String cacheNameB) { - GridQueryProcessor queryProc = grid(NODE_CLIENT).context().query(); - - List> r1 = queryProc.querySqlFieldsNoCache(new SqlFieldsQuery(updQry) - .setUpdateOnServer(true), true).getAll(); - - List> r2 = queryProc.querySqlFieldsNoCache(new SqlFieldsQuery(updQry) - .setSql(updQry.getSql().replace(cacheNameA, cacheNameB)) - .setUpdateOnServer(false), true).getAll(); - - assertNotNull(r1); - assertNotNull(r2); - - assertEquals(1, r1.size()); - assertEquals(1, r2.size()); - - assertEquals(r1.get(0).get(0), r2.get(0).get(0)); - - assertTrue(((Number)r1.get(0).get(0)).intValue() > 0); - } - - /** */ - private void compareTablesContent(String cacheNameA, String cacheNameB, String tableName) throws Exception { - IgniteEx client = grid(NODE_CLIENT); - - GridQueryProcessor queryProc = client.context().query(); - - assertEquals(client.cache(cacheNameA).size(), client.cache(cacheNameB).size()); - - int size = client.cache(cacheNameA).size(); - - String sqlText = "SELECT COUNT(*) FROM \"" + cacheNameA + "\"." + tableName + - " a JOIN \"" + cacheNameB + "\"." + tableName + " b ON a._key = b._key " + - "WHERE a._val = b._val"; - - List> r = queryProc.querySqlFieldsNoCache(new SqlFieldsQuery(sqlText), true).getAll(); - - assertNotNull(r); - - assertEquals(1, r.size()); - - assertEquals(size, ((Number)r.get(0).get(0)).intValue()); - } - - /** */ - private void checkResults(List> a, List> b) { - assertNotNull(a); - assertNotNull(b); - - assertEquals(a.size(), b.size()); - - assertTrue(a.size() > 0); - - int sz = a.size(); - for (int i = 0; i < sz; ++i) { - List ra = a.get(i); - List rb = b.get(i); - - assertNotNull(ra); - assertNotNull(rb); - - assertEquals(ra.size(), rb.size()); - - assertTrue(ra.size() > 0); - - int rSize = ra.size(); - - for (int j = 0; j < rSize; ++j) - assertEquals(ra.get(j), rb.get(j)); - } - } - - /** */ + /** + * Fills caches with initial data. + */ private void fillCaches() { Ignite client = grid(NODE_CLIENT); @@ -575,7 +535,14 @@ private void fillCaches() { } } - /** */ + /** + * Produces all possible combinations. + * + * @param a First array. + * @param b Second array. + * @param ends Endings array. + * @return Result. + */ private List produceCombination(String[] a, String[] b, String[] ends) { List res = new ArrayList<>(); @@ -606,12 +573,9 @@ private static class Organization { @QuerySqlField Date updated; - /** */ - public Organization() { - // No-op. - } - /** + * Constructor. + * * @param name Organization name. */ public Organization(String name, int rate) { @@ -632,7 +596,12 @@ public static class PersonKey { @QuerySqlField private Integer id; - /** */ + /** + * Constructor. + * + * @param orgId Organization id. + * @param id Person id. + */ PersonKey(int orgId, int id) { this.orgId = orgId; this.id = id; @@ -656,7 +625,13 @@ public static class Person { @QuerySqlField Date updated; - /** */ + /** + * Constructor. + * + * @param name Name. + * @param position Position. + * @param amount Amount. + */ private Person(String name, int position, int amount) { this.name = name; this.position = position; @@ -665,12 +640,12 @@ private Person(String name, int position, int amount) { this.updated = new Date(System.currentTimeMillis()); } - /** */ + /** {@inheritDoc} */ @Override public int hashCode() { return (name==null? 0: name.hashCode()) ^ position ^ amount ^ (updated == null ? 0 : updated.hashCode()); } - /** */ + /** {@inheritDoc} */ @Override public boolean equals(Object obj) { if (obj == null) return false; @@ -699,7 +674,13 @@ private static class Position { @QuerySqlField int rate; - /** */ + /** + * Constructor. + * + * @param id Id. + * @param name Name. + * @param rate Rate. + */ public Position(int id, String name, int rate) { this.id = id; this.name = name; @@ -707,13 +688,23 @@ public Position(int id, String name, int rate) { } } - /** */ + /** + * SQL function that always fails. + * + * @param param Arbitrary parameter. + * @return Result. + */ @QuerySqlFunction public static String Fail(String param) { throw new IgniteSQLException("Fail() called"); } - /** */ + /** + * SQL function that waits for condition. + * + * @param param Arbitrary parameter. + * @return Result. + */ @QuerySqlFunction public static String Wait(String param) { try { @@ -731,7 +722,13 @@ public static String Wait(String param) { return param; } - /** */ + /** + * SQL function that makes a concurrent modification. + * + * @param id Id. + * @param rate Rate. + * @return Result. + */ @QuerySqlFunction public static int Modify(final int id, final int rate) { try { From 5ba5ab5160f30b341980ae6b8307d542a6c0c100 Mon Sep 17 00:00:00 2001 From: tledkov-gridgain Date: Mon, 9 Oct 2017 18:34:45 +0300 Subject: [PATCH 25/40] IGNITE-6024: supports updateOnServer flag on JDBC --- .../jdbc2/JdbcConnectionSelfTest.java | 12 ++++- .../jdbc/suite/IgniteJdbcDriverTestSuite.java | 11 +++++ .../JdbcThinAbstractDmlStatementSelfTest.java | 14 ++++-- .../thin/JdbcThinComplexDmlDdlSelfTest.java | 10 +++- .../jdbc/thin/JdbcThinConnectionSelfTest.java | 18 +++++++- .../thin/JdbcThinInsertStatementSelfTest.java | 1 - ...inComplexDmlDdlUpdateOnServerSelfTest.java | 33 +++++++++++++ ...InsertStatementUpdateOnServerSelfTest.java | 33 +++++++++++++ ...nMergeStatementUpdateOnServerSelfTest.java | 33 +++++++++++++ ...UpdateStatementUpdateOnServerSelfTest.java | 33 +++++++++++++ .../org/apache/ignite/IgniteJdbcDriver.java | 9 +++- .../apache/ignite/IgniteJdbcThinDriver.java | 3 +- .../ignite/cache/query/SqlFieldsQuery.java | 40 ---------------- .../jdbc/thin/JdbcThinConnection.java | 4 +- .../internal/jdbc/thin/JdbcThinTcpIo.java | 15 +++++- .../internal/jdbc/thin/JdbcThinUtils.java | 6 +++ .../ignite/internal/jdbc2/JdbcConnection.java | 14 +++++- .../ignite/internal/jdbc2/JdbcQueryTask.java | 7 +++ .../internal/jdbc2/JdbcQueryTaskV3.java | 19 ++++++-- .../ignite/internal/jdbc2/JdbcResultSet.java | 2 +- .../internal/jdbc2/JdbcSqlFieldsQuery.java | 39 ++++++++++++++++ .../ignite/internal/jdbc2/JdbcStatement.java | 4 +- .../odbc/jdbc/JdbcConnectionContext.java | 7 ++- .../odbc/jdbc/JdbcRequestHandler.java | 10 +++- .../query/h2/DmlStatementsProcessor.java | 8 +++- .../IgniteSqlDistributedDmlFlagSelfTest.java | 46 +++++++++---------- .../IgniteSqlDistributedDmlSelfTest.java | 24 +++++----- 27 files changed, 357 insertions(+), 98 deletions(-) create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinComplexDmlDdlUpdateOnServerSelfTest.java create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinInsertStatementUpdateOnServerSelfTest.java create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinMergeStatementUpdateOnServerSelfTest.java create mode 100644 modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinUpdateStatementUpdateOnServerSelfTest.java diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java index aeb7c76113bd2..0940465e5f286 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java @@ -31,7 +31,6 @@ import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import static org.apache.ignite.IgniteJdbcDriver.CFG_URL_PREFIX; @@ -315,6 +314,7 @@ public void testSqlHints() throws Exception { assertFalse(((JdbcConnection)conn).isDistributedJoins()); assertFalse(((JdbcConnection)conn).isCollocatedQuery()); assertFalse(((JdbcConnection)conn).isLazy()); + assertFalse(((JdbcConnection)conn).updateOnServer()); } try (final Connection conn = DriverManager.getConnection(CFG_URL_PREFIX + "distributedJoins=true@" @@ -323,6 +323,7 @@ public void testSqlHints() throws Exception { assertTrue(((JdbcConnection)conn).isDistributedJoins()); assertFalse(((JdbcConnection)conn).isCollocatedQuery()); assertFalse(((JdbcConnection)conn).isLazy()); + assertFalse(((JdbcConnection)conn).updateOnServer()); } try (final Connection conn = DriverManager.getConnection(CFG_URL_PREFIX + "collocated=true@" @@ -331,6 +332,7 @@ public void testSqlHints() throws Exception { assertFalse(((JdbcConnection)conn).isDistributedJoins()); assertTrue(((JdbcConnection)conn).isCollocatedQuery()); assertFalse(((JdbcConnection)conn).isLazy()); + assertFalse(((JdbcConnection)conn).updateOnServer()); } try (final Connection conn = DriverManager.getConnection(CFG_URL_PREFIX + "lazy=true@" + configURL())) { @@ -338,6 +340,14 @@ public void testSqlHints() throws Exception { assertFalse(((JdbcConnection)conn).isDistributedJoins()); assertFalse(((JdbcConnection)conn).isCollocatedQuery()); assertTrue(((JdbcConnection)conn).isLazy()); + assertFalse(((JdbcConnection)conn).updateOnServer()); + } + try (final Connection conn = DriverManager.getConnection(CFG_URL_PREFIX + "updateOnServer=true@" + configURL())) { + assertFalse(((JdbcConnection)conn).isEnforceJoinOrder()); + assertFalse(((JdbcConnection)conn).isDistributedJoins()); + assertFalse(((JdbcConnection)conn).isCollocatedQuery()); + assertFalse(((JdbcConnection)conn).isLazy()); + assertTrue(((JdbcConnection)conn).updateOnServer()); } } } diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java index 1ae2427e0a0c9..f80fb615da1c8 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java @@ -58,6 +58,10 @@ import org.apache.ignite.jdbc.thin.JdbcThinSelectAfterAlterTable; import org.apache.ignite.jdbc.thin.JdbcThinStatementSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSelfTest; +import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinComplexDmlDdlUpdateOnServerSelfTest; +import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinInsertStatementUpdateOnServerSelfTest; +import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinMergeStatementUpdateOnServerSelfTest; +import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinUpdateStatementUpdateOnServerSelfTest; /** * JDBC driver test suite. @@ -152,6 +156,13 @@ public static TestSuite suite() throws Exception { suite.addTest(new TestSuite(JdbcThinSelectAfterAlterTable.class)); + // Update on server + suite.addTest(new TestSuite(JdbcThinInsertStatementUpdateOnServerSelfTest.class)); + suite.addTest(new TestSuite(JdbcThinUpdateStatementUpdateOnServerSelfTest.class)); + suite.addTest(new TestSuite(JdbcThinMergeStatementUpdateOnServerSelfTest.class)); + suite.addTest(new TestSuite(JdbcThinComplexDmlDdlUpdateOnServerSelfTest.class)); + + return suite; } } diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAbstractDmlStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAbstractDmlStatementSelfTest.java index afe5e2e4b79b1..69435daebda88 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAbstractDmlStatementSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAbstractDmlStatementSelfTest.java @@ -20,6 +20,7 @@ import java.io.Serializable; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.SQLException; import java.util.Collections; import org.apache.ignite.cache.QueryEntity; import org.apache.ignite.cache.query.annotations.QuerySqlField; @@ -42,9 +43,6 @@ public abstract class JdbcThinAbstractDmlStatementSelfTest extends JdbcThinAbstr /** IP finder. */ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); - /** URL. */ - private static final String URL = "jdbc:ignite:thin://127.0.0.1/"; - /** SQL SELECT query for verification. */ static final String SQL_SELECT = "select _key, id, firstName, lastName, age from Person"; @@ -67,7 +65,7 @@ public abstract class JdbcThinAbstractDmlStatementSelfTest extends JdbcThinAbstr @Override protected void beforeTest() throws Exception { ignite(0).getOrCreateCache(cacheConfig()); - conn = DriverManager.getConnection(URL); + conn = createConnection(); conn.setSchema('"' + DEFAULT_CACHE_NAME + '"'); } @@ -81,6 +79,14 @@ public abstract class JdbcThinAbstractDmlStatementSelfTest extends JdbcThinAbstr assertTrue(conn.isClosed()); } + /** + * @return JDBC connection. + * @throws SQLException On error. + */ + protected Connection createConnection() throws SQLException { + return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1/"); + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { return getConfiguration0(igniteInstanceName); diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinComplexDmlDdlSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinComplexDmlDdlSelfTest.java index 07601077a5a60..d4e03bc417908 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinComplexDmlDdlSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinComplexDmlDdlSelfTest.java @@ -93,6 +93,14 @@ private CacheConfiguration cacheConfiguration(@NotNull String name) throws Excep return cfg; } + /** + * @return JDBC connection. + * @throws SQLException On error. + */ + protected Connection createConnection() throws SQLException { + return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1"); + } + /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { super.beforeTestsStarted(); @@ -109,7 +117,7 @@ private CacheConfiguration cacheConfiguration(@NotNull String name) throws Excep @Override protected void beforeTest() throws Exception { super.beforeTest(); - conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1"); + conn = createConnection(); } /** {@inheritDoc} */ diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java index fbbec0d73f885..98a12e6a6943c 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java @@ -187,6 +187,7 @@ public void testSqlHints() throws Exception { assertFalse(io(conn).collocated()); assertFalse(io(conn).replicatedOnly()); assertFalse(io(conn).lazy()); + assertFalse(io(conn).updateOnServer()); } try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?distributedJoins=true")) { @@ -195,6 +196,7 @@ public void testSqlHints() throws Exception { assertFalse(io(conn).collocated()); assertFalse(io(conn).replicatedOnly()); assertFalse(io(conn).lazy()); + assertFalse(io(conn).updateOnServer()); } try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?enforceJoinOrder=true")) { @@ -203,6 +205,7 @@ public void testSqlHints() throws Exception { assertFalse(io(conn).collocated()); assertFalse(io(conn).replicatedOnly()); assertFalse(io(conn).lazy()); + assertFalse(io(conn).updateOnServer()); } try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?collocated=true")) { @@ -211,6 +214,7 @@ public void testSqlHints() throws Exception { assertTrue(io(conn).collocated()); assertFalse(io(conn).replicatedOnly()); assertFalse(io(conn).lazy()); + assertFalse(io(conn).updateOnServer()); } try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?replicatedOnly=true")) { @@ -219,6 +223,7 @@ public void testSqlHints() throws Exception { assertFalse(io(conn).collocated()); assertTrue(io(conn).replicatedOnly()); assertFalse(io(conn).lazy()); + assertFalse(io(conn).updateOnServer()); } try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?lazy=true")) { @@ -227,15 +232,26 @@ public void testSqlHints() throws Exception { assertFalse(io(conn).collocated()); assertFalse(io(conn).replicatedOnly()); assertTrue(io(conn).lazy()); + assertFalse(io(conn).updateOnServer()); + } + + try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?updateOnServer=true")) { + assertFalse(io(conn).distributedJoins()); + assertFalse(io(conn).enforceJoinOrder()); + assertFalse(io(conn).collocated()); + assertFalse(io(conn).replicatedOnly()); + assertFalse(io(conn).lazy()); + assertTrue(io(conn).updateOnServer()); } try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?distributedJoins=true&" + - "enforceJoinOrder=true&collocated=true&replicatedOnly=true&lazy=true")) { + "enforceJoinOrder=true&collocated=true&replicatedOnly=true&lazy=true&updateOnServer=true")) { assertTrue(io(conn).distributedJoins()); assertTrue(io(conn).enforceJoinOrder()); assertTrue(io(conn).collocated()); assertTrue(io(conn).replicatedOnly()); assertTrue(io(conn).lazy()); + assertTrue(io(conn).updateOnServer()); } } diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinInsertStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinInsertStatementSelfTest.java index 8ab5760e7c6f5..bf55da0879eb2 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinInsertStatementSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinInsertStatementSelfTest.java @@ -24,7 +24,6 @@ import java.util.Arrays; import java.util.HashSet; import java.util.concurrent.Callable; -import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.testframework.GridTestUtils; /** diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinComplexDmlDdlUpdateOnServerSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinComplexDmlDdlUpdateOnServerSelfTest.java new file mode 100644 index 0000000000000..6da222f769396 --- /dev/null +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinComplexDmlDdlUpdateOnServerSelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.jdbc.thin.updateonserver; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import org.apache.ignite.jdbc.thin.JdbcThinComplexDmlDdlSelfTest; + +/** + * Base class for complex SQL tests based on JDBC driver. + */ +public class JdbcThinComplexDmlDdlUpdateOnServerSelfTest extends JdbcThinComplexDmlDdlSelfTest { + /** {@inheritDoc} */ + @Override protected Connection createConnection() throws SQLException { + return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?updateOnServer=true"); + } +} \ No newline at end of file diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinInsertStatementUpdateOnServerSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinInsertStatementUpdateOnServerSelfTest.java new file mode 100644 index 0000000000000..0b4cec5579cf0 --- /dev/null +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinInsertStatementUpdateOnServerSelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.jdbc.thin.updateonserver; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import org.apache.ignite.jdbc.thin.JdbcThinInsertStatementSelfTest; + +/** + * Statement test. + */ +public class JdbcThinInsertStatementUpdateOnServerSelfTest extends JdbcThinInsertStatementSelfTest { + /** {@inheritDoc} */ + @Override protected Connection createConnection() throws SQLException { + return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?updateOnServer=true"); + } +} diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinMergeStatementUpdateOnServerSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinMergeStatementUpdateOnServerSelfTest.java new file mode 100644 index 0000000000000..b38eca3a3c785 --- /dev/null +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinMergeStatementUpdateOnServerSelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.jdbc.thin.updateonserver; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import org.apache.ignite.jdbc.thin.JdbcThinMergeStatementSelfTest; + +/** + * MERGE statement test. + */ +public class JdbcThinMergeStatementUpdateOnServerSelfTest extends JdbcThinMergeStatementSelfTest { + /** {@inheritDoc} */ + @Override protected Connection createConnection() throws SQLException { + return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?updateOnServer=true"); + } +} diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinUpdateStatementUpdateOnServerSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinUpdateStatementUpdateOnServerSelfTest.java new file mode 100644 index 0000000000000..9ab1fb3b7c1d4 --- /dev/null +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinUpdateStatementUpdateOnServerSelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.jdbc.thin.updateonserver; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSelfTest; + +/** + * + */ +public class JdbcThinUpdateStatementUpdateOnServerSelfTest extends JdbcThinUpdateStatementSelfTest { + /** {@inheritDoc} */ + @Override protected Connection createConnection() throws SQLException { + return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?updateOnServer=true"); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java index b03e38733f58b..2450817931d67 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java @@ -334,6 +334,9 @@ public class IgniteJdbcDriver implements Driver { /** Allow queries with multiple statements. */ private static final String PARAM_MULTIPLE_STMTS = "multipleStatementsAllowed"; + /** Server side update property name. */ + private static final String PARAM_UPDATE_ON_SERVER = "updateOnServer"; + /** Hostname property name. */ public static final String PROP_HOST = PROP_PREFIX + "host"; @@ -382,6 +385,9 @@ public class IgniteJdbcDriver implements Driver { /** Allow query with multiple statements. */ public static final String PROP_MULTIPLE_STMTS = PROP_PREFIX + PARAM_MULTIPLE_STMTS; + /** Server side update property name. */ + public static final String PROP_UPDATE_ON_SERVER = PROP_PREFIX + PARAM_UPDATE_ON_SERVER; + /** Cache name property name. */ public static final String PROP_CFG = PROP_PREFIX + "cfg"; @@ -454,7 +460,8 @@ public class IgniteJdbcDriver implements Driver { new JdbcDriverPropertyInfo("Enforce Join Order", info.getProperty(JdbcThinUtils.PROP_ENFORCE_JOIN_ORDER), ""), new JdbcDriverPropertyInfo("Lazy query execution", info.getProperty(JdbcThinUtils.PROP_LAZY), ""), new JdbcDriverPropertyInfo("Transactions Allowed", info.getProperty(PROP_TX_ALLOWED), ""), - new JdbcDriverPropertyInfo("Queries with multiple statements allowed", info.getProperty(PROP_MULTIPLE_STMTS), "") + new JdbcDriverPropertyInfo("Queries with multiple statements allowed", info.getProperty(PROP_MULTIPLE_STMTS), ""), + new JdbcDriverPropertyInfo("Server side update", info.getProperty(PROP_UPDATE_ON_SERVER), "") ); if (info.getProperty(PROP_CFG) != null) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcThinDriver.java b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcThinDriver.java index 8085ed42631cc..edf01bc8588f9 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcThinDriver.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcThinDriver.java @@ -186,7 +186,8 @@ public class IgniteJdbcThinDriver implements Driver { new JdbcDriverPropertyInfo("Enforce Join Order", info.getProperty(JdbcThinUtils.PROP_ENFORCE_JOIN_ORDER), ""), new JdbcDriverPropertyInfo("Collocated", info.getProperty(JdbcThinUtils.PROP_COLLOCATED), ""), new JdbcDriverPropertyInfo("Replicated only", info.getProperty(JdbcThinUtils.PROP_REPLICATED_ONLY), ""), - new JdbcDriverPropertyInfo("Lazy query execution flag", info.getProperty(JdbcThinUtils.PROP_LAZY),"") + new JdbcDriverPropertyInfo("Lazy query execution flag", info.getProperty(JdbcThinUtils.PROP_LAZY),""), + new JdbcDriverPropertyInfo("Execute update query on server", info.getProperty(JdbcThinUtils.PROP_UPDATE_ON_SERVER),"") ); return props.toArray(new DriverPropertyInfo[0]); diff --git a/modules/core/src/main/java/org/apache/ignite/cache/query/SqlFieldsQuery.java b/modules/core/src/main/java/org/apache/ignite/cache/query/SqlFieldsQuery.java index 50762f2c79571..2d128d158b9fb 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/query/SqlFieldsQuery.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/query/SqlFieldsQuery.java @@ -74,9 +74,6 @@ public class SqlFieldsQuery extends Query> { /** */ private boolean lazy; - /** Whether server side DML should be enabled. */ - private boolean updateOnServer; - /** Partitions for query */ private int[] parts; @@ -97,7 +94,6 @@ public SqlFieldsQuery(SqlFieldsQuery qry) { distributedJoins = qry.distributedJoins; replicatedOnly = qry.replicatedOnly; lazy = qry.lazy; - updateOnServer = qry.updateOnServer; parts = qry.parts; schema = qry.schema; } @@ -326,42 +322,6 @@ public boolean isLazy() { return lazy; } - /** - * Sets server side update flag. - *

- * By default, when processing DML command, Ignite first fetches all affected intermediate rows for analysis to the - * node which initiated the query and only then forms batches of updated values to be send to remote nodes. - * For simple DML commands (that however affect great deal of rows) such approach may be an overkill in terms of - * network delays and memory usage on initiating node. Use this flag as hint for Ignite to do all intermediate rows - * analysis and updates in place on corresponding remote data nodes. - *

- * There are limitations to what DML command can be optimized this way. The command containing LIMIT, OFFSET, - * DISTINCT, ORDER BY, GROUP BY, sub-query or UNION will be processed the usual way despite this flag setting. - *

- * Defaults to {@code false}, meaning that intermediate results will be fetched to initiating node first. - * Only affects DML commands. Ignored when {@link #isLocal()} is {@code true}. - * Note that when set to {@code true}, the query may fail in the case of even single node failure. - * - * @param updateOnServer - * @return {@code this} For chaining. - */ - public SqlFieldsQuery setUpdateOnServer(boolean updateOnServer) { - this.updateOnServer = updateOnServer; - - return this; - } - - /** - * Gets server side update flag. - *

- * See {@link #setUpdateOnServer(boolean)} for more information. - * - * @return Server side update flag. - */ - public boolean isUpdateOnServer() { - return updateOnServer; - } - /** * Gets partitions for query, in ascending order. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java index 5afed4e5bc5a1..9c4b8278985d3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java @@ -62,6 +62,7 @@ import static org.apache.ignite.internal.jdbc.thin.JdbcThinUtils.PROP_SOCK_RCV_BUF; import static org.apache.ignite.internal.jdbc.thin.JdbcThinUtils.PROP_SOCK_SND_BUF; import static org.apache.ignite.internal.jdbc.thin.JdbcThinUtils.PROP_TCP_NO_DELAY; +import static org.apache.ignite.internal.jdbc.thin.JdbcThinUtils.PROP_UPDATE_ON_SERVER; /** * JDBC connection implementation. @@ -136,10 +137,11 @@ public JdbcThinConnection(String url, Properties props, String schema) throws SQ int sockRcvBuf = extractIntNonNegative(props, PROP_SOCK_RCV_BUF, 0); boolean tcpNoDelay = extractBoolean(props, PROP_TCP_NO_DELAY, true); + boolean updateOnServer = extractBoolean(props, PROP_UPDATE_ON_SERVER, false); try { cliIo = new JdbcThinTcpIo(host, port, distributedJoins, enforceJoinOrder, collocated, replicatedOnly, - autoCloseServerCursor, lazyExec, sockSndBuf, sockRcvBuf, tcpNoDelay); + autoCloseServerCursor, lazyExec, sockSndBuf, sockRcvBuf, tcpNoDelay, updateOnServer); cliIo.start(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java index 9e12fbf571005..0ffb2d46352b8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java @@ -100,6 +100,9 @@ public class JdbcThinTcpIo { /** Flag to automatically close server cursor. */ private final boolean autoCloseServerCursor; + /** Executes update queries on server nodes distributedly. */ + private final boolean updateOnServer; + /** Socket send buffer. */ private final int sockSndBuf; @@ -138,10 +141,11 @@ public class JdbcThinTcpIo { * @param sockSndBuf Socket send buffer. * @param sockRcvBuf Socket receive buffer. * @param tcpNoDelay TCP no delay flag. + * @param updateOnServer Executes update queries on ignite server nodes distributedly. */ JdbcThinTcpIo(String host, int port, boolean distributedJoins, boolean enforceJoinOrder, boolean collocated, boolean replicatedOnly, boolean autoCloseServerCursor, boolean lazy, int sockSndBuf, int sockRcvBuf, - boolean tcpNoDelay) { + boolean tcpNoDelay, boolean updateOnServer) { this.host = host; this.port = port; this.distributedJoins = distributedJoins; @@ -153,6 +157,7 @@ public class JdbcThinTcpIo { this.sockSndBuf = sockSndBuf; this.sockRcvBuf = sockRcvBuf; this.tcpNoDelay = tcpNoDelay; + this.updateOnServer = updateOnServer; } /** @@ -211,6 +216,7 @@ public void handshake(ClientListenerProtocolVersion ver) throws IOException, SQL writer.writeBoolean(replicatedOnly); writer.writeBoolean(autoCloseServerCursor); writer.writeBoolean(lazy); + writer.writeBoolean(updateOnServer); send(writer.array()); @@ -491,4 +497,11 @@ IgniteProductVersion igniteVersion() { public boolean lazy() { return lazy; } + + /** + * @return Server side update flag. + */ + public boolean updateOnServer() { + return updateOnServer; + } } \ No newline at end of file diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinUtils.java index 52b3abcec2fc2..12fe25face637 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinUtils.java @@ -81,6 +81,9 @@ public class JdbcThinUtils { /** Parameter: Automatically close server cursor. */ public static final String PARAM_AUTO_CLOSE_SERVER_CURSOR = "autoCloseServerCursor"; + /** Parameter: execute update query in distributed mode on ignite server nodes. */ + public static final String PARAM_UPDATE_ON_SERVER = "updateOnServer"; + /** Distributed joins property name. */ public static final String PROP_DISTRIBUTED_JOINS = PROP_PREFIX + PARAM_DISTRIBUTED_JOINS; @@ -108,6 +111,9 @@ public class JdbcThinUtils { /** Automatically close server cursor. */ public static final String PROP_AUTO_CLOSE_SERVER_CURSORS = PROP_PREFIX + PARAM_AUTO_CLOSE_SERVER_CURSOR; + /** Executes update queries on ignite server nodes in distributed mode. */ + public static final String PROP_UPDATE_ON_SERVER = PROP_PREFIX + PARAM_UPDATE_ON_SERVER; + /** Default port. */ public static final int DFLT_PORT = ClientConnectorConfiguration.DFLT_PORT; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java index ccc09ece9a4bd..4e48ae5684191 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java @@ -82,12 +82,13 @@ import static org.apache.ignite.IgniteJdbcDriver.PROP_LOCAL; import static org.apache.ignite.IgniteJdbcDriver.PROP_MULTIPLE_STMTS; import static org.apache.ignite.IgniteJdbcDriver.PROP_NODE_ID; -import static org.apache.ignite.IgniteJdbcDriver.PROP_TX_ALLOWED; import static org.apache.ignite.IgniteJdbcDriver.PROP_STREAMING; import static org.apache.ignite.IgniteJdbcDriver.PROP_STREAMING_ALLOW_OVERWRITE; import static org.apache.ignite.IgniteJdbcDriver.PROP_STREAMING_FLUSH_FREQ; import static org.apache.ignite.IgniteJdbcDriver.PROP_STREAMING_PER_NODE_BUF_SIZE; import static org.apache.ignite.IgniteJdbcDriver.PROP_STREAMING_PER_NODE_PAR_OPS; +import static org.apache.ignite.IgniteJdbcDriver.PROP_TX_ALLOWED; +import static org.apache.ignite.IgniteJdbcDriver.PROP_UPDATE_ON_SERVER; import static org.apache.ignite.internal.jdbc2.JdbcUtils.convertToSqlException; import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.createJdbcSqlException; @@ -168,6 +169,9 @@ public class JdbcConnection implements Connection { /** Allow queries with multiple statements. */ private final boolean multipleStmts; + /** Update on server flag. */ + private final boolean updateOnServer; + /** Statements. */ final Set statements = new HashSet<>(); @@ -209,6 +213,7 @@ public JdbcConnection(String url, Properties props) throws SQLException { streamNodeParOps = Integer.parseInt(props.getProperty(PROP_STREAMING_PER_NODE_PAR_OPS, "0")); multipleStmts = Boolean.parseBoolean(props.getProperty(PROP_MULTIPLE_STMTS)); + updateOnServer = Boolean.parseBoolean(props.getProperty(PROP_UPDATE_ON_SERVER)); String nodeIdProp = props.getProperty(PROP_NODE_ID); @@ -853,6 +858,13 @@ boolean isMultipleStatementsAllowed() { return multipleStmts; } + /** + * @return {@code true} if update on server is enabled, {@code false} otherwise. + */ + boolean updateOnServer() { + return updateOnServer; + } + /** * @return Local query flag. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java index ecbfb713451bd..516de1cda9a20 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java @@ -240,6 +240,13 @@ protected boolean updateMetadata() { return false; } + /** + * @return Flag to update enable server side updates. + */ + protected boolean updateOnServer() { + return false; + } + /** * Schedules removal of stored cursor in case of remote query execution. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV3.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV3.java index cb2d45220f7f3..986b3819dde19 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV3.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV3.java @@ -30,6 +30,9 @@ class JdbcQueryTaskV3 extends JdbcQueryTaskV2 { /** Update metadata on demand flag. */ private final boolean updateMeta; + /** Update metadata on demand flag. */ + private final boolean updateOnServer; + /** * @param ignite Ignite. * @param cacheName Cache name. @@ -46,14 +49,16 @@ class JdbcQueryTaskV3 extends JdbcQueryTaskV2 { * @param enforceJoinOrder Enforce joins order flag. * @param lazy Lazy query execution flag. * @param updateMeta Update metadata on demand. + * @param updateOnServer Flkag to enable server side updates. */ public JdbcQueryTaskV3(Ignite ignite, String cacheName, String schemaName, String sql, Boolean isQry, boolean loc, Object[] args, int fetchSize, UUID uuid, boolean locQry, boolean collocatedQry, boolean distributedJoins, - boolean enforceJoinOrder, boolean lazy, boolean updateMeta) { + boolean enforceJoinOrder, boolean lazy, boolean updateMeta, boolean updateOnServer) { super(ignite, cacheName, schemaName, sql, isQry, loc, args, fetchSize, uuid, locQry, collocatedQry, distributedJoins, enforceJoinOrder, lazy); this.updateMeta = updateMeta; + this.updateOnServer = updateOnServer; } /** {@inheritDoc} */ @@ -61,6 +66,11 @@ public JdbcQueryTaskV3(Ignite ignite, String cacheName, String schemaName, Strin return updateMeta; } + /** {@inheritDoc} */ + @Override protected boolean updateOnServer() { + return updateOnServer; + } + /** * @param ignite Ignite. * @param cacheName Cache name. @@ -77,16 +87,17 @@ public JdbcQueryTaskV3(Ignite ignite, String cacheName, String schemaName, Strin * @param enforceJoinOrder Enforce joins order flag. * @param lazy Lazy query execution flag. * @param updateMeta Update metadata on demand. + * @param updateOnServer Update on server flag. * @return Appropriate task JdbcQueryTask or JdbcQueryTaskV2. */ public static JdbcQueryTask createTask(Ignite ignite, String cacheName, String schemaName, String sql, Boolean isQry, boolean loc, Object[] args, int fetchSize, UUID uuid, boolean locQry, boolean collocatedQry, boolean distributedJoins, - boolean enforceJoinOrder, boolean lazy, boolean updateMeta) { + boolean enforceJoinOrder, boolean lazy, boolean updateMeta, boolean updateOnServer) { - if (updateMeta) + if (updateMeta || updateOnServer) return new JdbcQueryTaskV3(ignite, cacheName, schemaName, sql, isQry, loc, args, fetchSize, - uuid, locQry, collocatedQry, distributedJoins, enforceJoinOrder, lazy, true); + uuid, locQry, collocatedQry, distributedJoins, enforceJoinOrder, lazy, updateMeta, updateOnServer); else return JdbcQueryTaskV2.createTask(ignite, cacheName, schemaName, sql, isQry, loc, args, fetchSize, uuid, locQry, collocatedQry, distributedJoins, enforceJoinOrder, lazy); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java index 69d4252d7bcee..e2ff5d866dffb 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcResultSet.java @@ -205,7 +205,7 @@ private void fetchPage() throws SQLException { // Connections from new clients send queries with new tasks, so we have to continue in the same manner JdbcQueryTask qryTask = JdbcQueryTaskV3.createTask(loc ? ignite : null, conn.cacheName(), conn.schemaName(), null,true, loc, null, fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(), - conn.isDistributedJoins(), conn.isEnforceJoinOrder(), conn.isLazy(), updateMetadata); + conn.isDistributedJoins(), conn.isEnforceJoinOrder(), conn.isLazy(), updateMetadata, false); try { JdbcQueryTaskResult res = diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcSqlFieldsQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcSqlFieldsQuery.java index d8b9a2658398d..fec1ee8fe20ae 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcSqlFieldsQuery.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcSqlFieldsQuery.java @@ -31,6 +31,9 @@ public final class JdbcSqlFieldsQuery extends SqlFieldsQuery { /** Flag set by JDBC driver to enforce checks for correct operation type. */ private final boolean isQry; + /** Whether server side DML should be enabled. */ + private boolean updateOnServer; + /** * @param sql SQL query. * @param isQry Flag indicating whether this object denotes a query or an update operation. @@ -102,4 +105,40 @@ public boolean isQuery() { return this; } + + /** + * Sets server side update flag. + *

+ * By default, when processing DML command, Ignite first fetches all affected intermediate rows for analysis to the + * node which initiated the query and only then forms batches of updated values to be send to remote nodes. + * For simple DML commands (that however affect great deal of rows) such approach may be an overkill in terms of + * network delays and memory usage on initiating node. Use this flag as hint for Ignite to do all intermediate rows + * analysis and updates in place on corresponding remote data nodes. + *

+ * There are limitations to what DML command can be optimized this way. The command containing LIMIT, OFFSET, + * DISTINCT, ORDER BY, GROUP BY, sub-query or UNION will be processed the usual way despite this flag setting. + *

+ * Defaults to {@code false}, meaning that intermediate results will be fetched to initiating node first. + * Only affects DML commands. Ignored when {@link #isLocal()} is {@code true}. + * Note that when set to {@code true}, the query may fail in the case of even single node failure. + * + * @param updateOnServer Server side update flag. + * @return {@code this} For chaining. + */ + public SqlFieldsQuery setUpdateOnServer(boolean updateOnServer) { + this.updateOnServer = updateOnServer; + + return this; + } + + /** + * Gets server side update flag. + *

+ * See {@link #setUpdateOnServer(boolean)} for more information. + * + * @return Server side update flag. + */ + public boolean isUpdateOnServer() { + return updateOnServer; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java index acac12337d35a..b1798d8f91b55 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java @@ -161,9 +161,9 @@ private void executeSingle(String sql, Boolean isQuery) throws SQLException { else isQuery = true; - JdbcQueryTask qryTask = JdbcQueryTaskV2.createTask(loc ? ignite : null, conn.cacheName(), conn.schemaName(), + JdbcQueryTask qryTask = JdbcQueryTaskV3.createTask(loc ? ignite : null, conn.cacheName(), conn.schemaName(), sql, isQuery, loc, getArgs(), fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(), - conn.isDistributedJoins(), conn.isEnforceJoinOrder(), conn.isLazy()); + conn.isDistributedJoins(), conn.isEnforceJoinOrder(), conn.isLazy(), false, conn.updateOnServer()); try { JdbcQueryTaskResult qryRes = diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java index a6a7aa5c27507..da71be6510a07 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java @@ -104,8 +104,13 @@ public JdbcConnectionContext(GridKernalContext ctx, GridSpinBusyLock busyLock, i if (ver.compareTo(VER_2_1_5) >= 0) lazyExec = reader.readBoolean(); + boolean updateOnServer = false; + + if (ver.compareTo(VER_2_3_0) >= 0) + updateOnServer = reader.readBoolean(); + handler = new JdbcRequestHandler(ctx, busyLock, maxCursors, distributedJoins, enforceJoinOrder, - collocated, replicatedOnly, autoCloseCursors, lazyExec, ver); + collocated, replicatedOnly, autoCloseCursors, lazyExec, updateOnServer, ver); parser = new JdbcMessageParser(ctx); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java index 166402fcce2c0..e25d8e24f7a6f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java @@ -103,6 +103,9 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler { /** Lazy query execution flag. */ private final boolean lazy; + /** Update on server flag. */ + private final boolean updateOnServer; + /** Automatic close of cursors. */ private final boolean autoCloseCursors; @@ -121,11 +124,12 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler { * @param replicatedOnly Replicated only flag. * @param autoCloseCursors Flag to automatically close server cursors. * @param lazy Lazy query execution flag. + * @param updateOnServer Server side update flag. * @param protocolVer Protocol version. */ public JdbcRequestHandler(GridKernalContext ctx, GridSpinBusyLock busyLock, int maxCursors, boolean distributedJoins, boolean enforceJoinOrder, boolean collocated, boolean replicatedOnly, - boolean autoCloseCursors, boolean lazy, ClientListenerProtocolVersion protocolVer) { + boolean autoCloseCursors, boolean lazy, boolean updateOnServer, ClientListenerProtocolVersion protocolVer) { this.ctx = ctx; this.busyLock = busyLock; this.maxCursors = maxCursors; @@ -135,6 +139,7 @@ public JdbcRequestHandler(GridKernalContext ctx, GridSpinBusyLock busyLock, int this.replicatedOnly = replicatedOnly; this.autoCloseCursors = autoCloseCursors; this.lazy = lazy; + this.updateOnServer = updateOnServer; this.protocolVer = protocolVer; log = ctx.log(getClass()); @@ -271,6 +276,9 @@ private JdbcResponse executeQuery(JdbcQueryExecuteRequest req) { assert req.expectedStatementType() == JdbcStatementType.UPDATE_STMT_TYPE; qry = new JdbcSqlFieldsQuery(sql, false); + + if (updateOnServer) + ((JdbcSqlFieldsQuery)qry).setUpdateOnServer(true); } qry.setArgs(req.arguments()); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 334a8cc8fe224..ef39a84cb9487 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -50,6 +50,7 @@ import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.jdbc2.JdbcSqlFieldsQuery; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheOperationContext; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; @@ -412,7 +413,8 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo return doFastUpdate(plan, fieldsQry.getArgs()); } - if (plan.distributed && !loc && !fieldsQry.isLocal() && fieldsQry.isUpdateOnServer()) { + if (plan.distributed && !loc && !fieldsQry.isLocal() && + fieldsQry instanceof JdbcSqlFieldsQuery && ((JdbcSqlFieldsQuery)fieldsQry).isUpdateOnServer()) { UpdateResult result = doDistributedUpdate(schemaName, fieldsQry, plan, cancel); // null is returned in case not all nodes support distributed DML. @@ -511,7 +513,8 @@ private UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared res = UpdatePlanBuilder.planForStatement(p, errKeysPos); - if (fieldsQry.isUpdateOnServer() && !loc && !F.isEmpty(res.selectQry)) + if (fieldsQry instanceof JdbcSqlFieldsQuery && ((JdbcSqlFieldsQuery)fieldsQry).isUpdateOnServer() + && !loc && !F.isEmpty(res.selectQry)) checkPlanCanBeDistributed(fieldsQry, conn, res); // Don't cache re-runs @@ -523,6 +526,7 @@ private UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared /** * Perform single cache operation based on given args. + * @param plan Update plan. * @param args Query parameters. * @return 1 if an item was affected, 0 otherwise. * @throws IgniteCheckedException if failed. diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java index 3db44e83e7220..96708bf2ce01d 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java @@ -9,17 +9,17 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.QueryEntity; -import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.jdbc2.JdbcSqlFieldsQuery; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -/** Tests for {@link SqlFieldsQuery#updateOnServer} flag. */ +/** Tests for {@link JdbcSqlFieldsQuery#updateOnServer} flag. */ public class IgniteSqlDistributedDmlFlagSelfTest extends GridCommonAbstractTest { /** IP finder. */ private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); @@ -183,7 +183,7 @@ public void testUpdate() throws Exception { String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE depo > 0"; - checkUpdate(client.cache(CACHE_ACCOUNT), accounts, new SqlFieldsQuery(text).setArgs(10)); + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, new JdbcSqlFieldsQuery(text, false).setArgs(10)); } /** @@ -196,7 +196,7 @@ public void testUpdateFastKey() throws Exception { String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE _key = ?"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQuery(text).setArgs(10, 1)); + new JdbcSqlFieldsQuery(text, false).setArgs(10, 1)); } /** @@ -209,7 +209,7 @@ public void testUpdateLimit() throws Exception { String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE sn >= ? AND sn < ? LIMIT ?"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQuery(text).setArgs(10, 0, 10, 10)); + new JdbcSqlFieldsQuery(text, false).setArgs(10, 0, 10, 10)); } /** @@ -227,7 +227,7 @@ public void testUpdateWhereSubquery() throws Exception { "WHERE accountId IN (SELECT p._key FROM \"acc\".Account p WHERE depo < ?)"; checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQuery(text).setArgs(0, 0)); + new JdbcSqlFieldsQuery(text, false).setArgs(0, 0)); } /** @@ -244,7 +244,7 @@ public void testUpdateSetSubquery() throws Exception { "(SELECT a.depo/t.price FROM \"acc\".Account a WHERE t.accountId = a._key)"; checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQuery(text)); + new JdbcSqlFieldsQuery(text, false)); } /** @@ -261,7 +261,7 @@ public void testUpdateSetTableSubquery() throws Exception { "(SELECT a.depo/t.price FROM \"acc\".Account a WHERE t.accountId = a._key)"; checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQuery(text)); + new JdbcSqlFieldsQuery(text, false)); } /** @@ -273,7 +273,7 @@ public void testInsertValues() throws Exception { " VALUES (?, ?, ?, ?), (?, ?, ?, ?)"; checkUpdate(client.cache(CACHE_ACCOUNT), null, - new SqlFieldsQuery(text).setArgs(1, "John Marry", 11111, 100, 2, "Marry John", 11112, 200)); + new JdbcSqlFieldsQuery(text, false).setArgs(1, "John Marry", 11111, 100, 2, "Marry John", 11112, 200)); } /** @@ -289,7 +289,7 @@ public void testInsertFromSelect() throws Exception { "SELECT a._key, a._key, ?, a.depo/?, ? FROM \"acc\".Account a"; checkUpdate(client.cache(CACHE_TRADE), null, - new SqlFieldsQuery(text).setArgs(1, 10, 10)); + new JdbcSqlFieldsQuery(text, false).setArgs(1, 10, 10)); } /** @@ -306,7 +306,7 @@ public void testInsertFromSelectOrderBy() throws Exception { "ORDER BY a.sn DESC"; checkUpdate(client.cache(CACHE_TRADE), null, - new SqlFieldsQuery(text).setArgs(1, 10, 10)); + new JdbcSqlFieldsQuery(text, false).setArgs(1, 10, 10)); } /** @@ -324,7 +324,7 @@ public void testInsertFromSelectUnion() throws Exception { "SELECT 101 + a2._key, a2._key, 1, a2.depo, 1 FROM \"acc\".Account a2"; checkUpdate(client.cache(CACHE_TRADE), null, - new SqlFieldsQuery(text)); + new JdbcSqlFieldsQuery(text, false)); } /** @@ -344,7 +344,7 @@ public void testInsertFromSelectGroupBy() throws Exception { "GROUP BY accountId"; checkUpdate(client.cache(CACHE_REPORT), null, - new SqlFieldsQuery(text)); + new JdbcSqlFieldsQuery(text, false)); } /** @@ -360,7 +360,7 @@ public void testInsertFromSelectDistinct() throws Exception { "SELECT DISTINCT sn, name FROM \"acc\".Account "; checkUpdate(client.cache(CACHE_LIST), null, - new SqlFieldsQuery(text)); + new JdbcSqlFieldsQuery(text, false)); } /** @@ -379,7 +379,7 @@ public void testInsertFromSelectJoin() throws Exception { "FROM \"acc\".Account a JOIN \"stock\".Stock s ON 1=1"; checkUpdate(client.cache(CACHE_TRADE), null, - new SqlFieldsQuery(text).setArgs(10, 10)); + new JdbcSqlFieldsQuery(text, false).setArgs(10, 10)); } /** @@ -394,7 +394,7 @@ public void testDelete() throws Exception { String text = "DELETE FROM \"acc\".Account WHERE sn > ?"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQuery(text).setArgs(10)); + new JdbcSqlFieldsQuery(text, false).setArgs(10)); } /** @@ -409,7 +409,7 @@ public void testDeleteTop() throws Exception { String text = "DELETE TOP ? FROM \"acc\".Account WHERE sn < ?"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQuery(text).setArgs(10, 10)); + new JdbcSqlFieldsQuery(text, false).setArgs(10, 10)); } /** @@ -427,7 +427,7 @@ public void testDeleteWhereSubquery() throws Exception { "WHERE _key IN (SELECT t.accountId FROM \"trade\".Trade t)"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQuery(text)); + new JdbcSqlFieldsQuery(text, false)); } /** @@ -441,7 +441,7 @@ public void testMergeValues() throws Exception { " VALUES (?, ?, ?, ?), (?, ?, ?, ?)"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQuery(text).setArgs(0, "John Marry", 11111, 100, 1, "Marry John", 11112, 200)); + new JdbcSqlFieldsQuery(text, false).setArgs(0, "John Marry", 11111, 100, 1, "Marry John", 11112, 200)); } /** @@ -464,7 +464,7 @@ public void testMergeFromSelectJoin() throws Exception { "FROM \"acc\".Account a JOIN \"stock\".Stock s ON 1=1"; checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQuery(text).setArgs(10, 10)); + new JdbcSqlFieldsQuery(text, false).setArgs(10, 10)); } /** @@ -485,7 +485,7 @@ public void testMergeFromSelectOrderBy() throws Exception { "ORDER BY a.sn DESC"; checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQuery(text).setArgs(1, 10, 10)); + new JdbcSqlFieldsQuery(text, false).setArgs(1, 10, 10)); } /** @@ -509,7 +509,7 @@ public void testMergeFromSelectGroupBy() throws Exception { "GROUP BY accountId"; checkUpdate(client.cache(CACHE_REPORT), reports, - new SqlFieldsQuery(text)); + new JdbcSqlFieldsQuery(text, false)); } /** @@ -580,7 +580,7 @@ private Map getTrades(int numAccounts, int numStocks) { * @param Key type. * @param Value type. */ - private void checkUpdate(IgniteCache cache, Map initial, SqlFieldsQuery qry) { + private void checkUpdate(IgniteCache cache, Map initial, JdbcSqlFieldsQuery qry) { cache.clear(); if (!F.isEmpty(initial)) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java index e973ce2833e1f..70d3fd03c7676 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java @@ -42,6 +42,7 @@ import org.apache.ignite.events.CacheQueryExecutedEvent; import org.apache.ignite.events.Event; import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.jdbc2.JdbcSqlFieldsQuery; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor; import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor; @@ -209,7 +210,7 @@ public void testSimpleUpdateDistributedReplicated() throws Exception { Position p = cache.get(1); - List> r = cache.query(new SqlFieldsQuery("UPDATE Position p SET name = CONCAT('A ', name)") + List> r = cache.query(new JdbcSqlFieldsQuery("UPDATE Position p SET name = CONCAT('A ', name)", false) .setUpdateOnServer(true)).getAll(); assertEquals((long)cache.size(), r.get(0).get(0)); @@ -226,8 +227,8 @@ public void testSimpleUpdateDistributedPartitioned() throws Exception { IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); - List> r = cache.query(new SqlFieldsQuery( - "UPDATE Person SET position = CASEWHEN(position = 1, 1, position - 1)") + List> r = cache.query(new JdbcSqlFieldsQuery( + "UPDATE Person SET position = CASEWHEN(position = 1, 1, position - 1)", false) .setUpdateOnServer(true)).getAll(); assertEquals((long)cache.size(), r.get(0).get(0)); @@ -245,7 +246,7 @@ public void testDistributedUpdateFailedKeys() throws Exception { GridTestUtils.assertThrows(log, new Callable() { @Override public Object call() { - return cache.query(new SqlFieldsQuery("UPDATE Organization SET rate = Modify(_key, rate - 1)") + return cache.query(new JdbcSqlFieldsQuery("UPDATE Organization SET rate = Modify(_key, rate - 1)", false) .setUpdateOnServer(true)); } }, CacheException.class, "Failed to update some keys because they had been modified concurrently"); @@ -262,7 +263,7 @@ public void testDistributedUpdateFail() throws Exception { GridTestUtils.assertThrows(log, new Callable() { @Override public Object call() { - return cache.query(new SqlFieldsQuery("UPDATE Person SET name = Fail(name)") + return cache.query(new JdbcSqlFieldsQuery("UPDATE Person SET name = Fail(name)", false) .setUpdateOnServer(true)); } }, CacheException.class, "Failed to execute SQL query"); @@ -285,8 +286,8 @@ public void testQueryParallelism() throws Exception { for (int i = 0; i < 1024; i++) cache.put(i, new Organization("Acme Inc #" + i, 0)); - List> r = cache.query(new SqlFieldsQuery("UPDATE \"" + cacheName + - "\".Organization o SET name = UPPER(name)").setUpdateOnServer(true)).getAll(); + List> r = cache.query(new JdbcSqlFieldsQuery("UPDATE \"" + cacheName + + "\".Organization o SET name = UPPER(name)", false).setUpdateOnServer(true)).getAll(); assertEquals((long)cache.size(), r.get(0).get(0)); } @@ -320,7 +321,7 @@ public void testEvents() throws Exception { for (int i = 0; i < 1024; i++) cache.put(i, new Organization("Acme Inc #" + i, 0)); - cache.query(new SqlFieldsQuery("UPDATE \"org\".Organization o SET name = UPPER(name)") + cache.query(new JdbcSqlFieldsQuery("UPDATE \"org\".Organization o SET name = UPPER(name)", false) .setUpdateOnServer(true)).getAll(); assertTrue(latch.await(5000, MILLISECONDS)); @@ -347,7 +348,7 @@ public void testSpecificPartitionsUpdate() throws Exception { IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); // UPDATE over even partitions - cache.query(new SqlFieldsQuery("UPDATE Person SET position = 0") + cache.query(new JdbcSqlFieldsQuery("UPDATE Person SET position = 0", false) .setUpdateOnServer(true) .setPartitions(parts)); @@ -375,7 +376,7 @@ public void testCancel() throws Exception { final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { @Override public Object call() { - return cache.query(new SqlFieldsQuery("UPDATE Organization SET name = WAIT(name)") + return cache.query(new JdbcSqlFieldsQuery("UPDATE Organization SET name = WAIT(name)", false) .setUpdateOnServer(true)); } }); @@ -421,7 +422,7 @@ public void testNodeStopDuringUpdate() throws Exception { final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { @Override public Object call() { - return cache.query(new SqlFieldsQuery("UPDATE Organization SET name = WAIT(name)") + return cache.query(new JdbcSqlFieldsQuery("UPDATE Organization SET name = WAIT(name)", false) .setUpdateOnServer(true)); } }); @@ -577,6 +578,7 @@ private static class Organization { * Constructor. * * @param name Organization name. + * @param rate Rate. */ public Organization(String name, int rate) { this.name = name; From 8257f4a1cee3cfa832c8c582f1f2f036103ffc12 Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Tue, 10 Oct 2017 16:27:52 +0300 Subject: [PATCH 26/40] IGNITE-6024: Renamed JdbcSqlFieldsQuery to SqlFieldsQueryEx --- .../internal/jdbc2/JdbcBatchUpdateTask.java | 2 +- .../JdbcQueryMultipleStatementsTask.java | 2 +- .../ignite/internal/jdbc2/JdbcQueryTask.java | 2 +- ...FieldsQuery.java => SqlFieldsQueryEx.java} | 24 +++++----- .../odbc/jdbc/JdbcRequestHandler.java | 10 ++-- .../odbc/odbc/OdbcConnectionContext.java | 13 +++++- .../odbc/odbc/OdbcRequestHandler.java | 15 ++++-- .../resources/META-INF/classnames.properties | 4 +- .../query/h2/DmlStatementsProcessor.java | 6 +-- .../processors/query/h2/IgniteH2Indexing.java | 5 +- .../IgniteSqlDistributedDmlFlagSelfTest.java | 46 +++++++++---------- .../IgniteSqlDistributedDmlSelfTest.java | 20 ++++---- 12 files changed, 83 insertions(+), 66 deletions(-) rename modules/core/src/main/java/org/apache/ignite/internal/jdbc2/{JdbcSqlFieldsQuery.java => SqlFieldsQueryEx.java} (84%) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBatchUpdateTask.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBatchUpdateTask.java index e4916f70c7891..9bf859d9cb4d2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBatchUpdateTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBatchUpdateTask.java @@ -162,7 +162,7 @@ public JdbcBatchUpdateTask(Ignite ignite, String cacheName, String schemaName, S * @throws SQLException If failed. */ private Integer doSingleUpdate(IgniteCache cache, String sqlText, List args) throws SQLException { - SqlFieldsQuery qry = new JdbcSqlFieldsQuery(sqlText, false); + SqlFieldsQuery qry = new SqlFieldsQueryEx(sqlText, false); qry.setPageSize(fetchSize); qry.setLocal(locQry); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryMultipleStatementsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryMultipleStatementsTask.java index bf7c24e640ccd..0ff51ae3405cf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryMultipleStatementsTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryMultipleStatementsTask.java @@ -109,7 +109,7 @@ public JdbcQueryMultipleStatementsTask(Ignite ignite, String schemaName, String /** {@inheritDoc} */ @Override public List call() throws Exception { - SqlFieldsQuery qry = (isQry != null ? new JdbcSqlFieldsQuery(sql, isQry) : new SqlFieldsQuery(sql)) + SqlFieldsQuery qry = (isQry != null ? new SqlFieldsQueryEx(sql, isQry) : new SqlFieldsQuery(sql)) .setArgs(args); qry.setPageSize(fetchSize); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java index 516de1cda9a20..7e0bdddd30779 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java @@ -156,7 +156,7 @@ public JdbcQueryTask(Ignite ignite, String cacheName, String schemaName, String throw new SQLException("Cache not found [cacheName=" + cacheName + ']'); } - SqlFieldsQuery qry = (isQry != null ? new JdbcSqlFieldsQuery(sql, isQry) : new SqlFieldsQuery(sql)) + SqlFieldsQuery qry = (isQry != null ? new SqlFieldsQueryEx(sql, isQry) : new SqlFieldsQuery(sql)) .setArgs(args); qry.setPageSize(fetchSize); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcSqlFieldsQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/SqlFieldsQueryEx.java similarity index 84% rename from modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcSqlFieldsQuery.java rename to modules/core/src/main/java/org/apache/ignite/internal/jdbc2/SqlFieldsQueryEx.java index fec1ee8fe20ae..0d003d5acdc0c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcSqlFieldsQuery.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/SqlFieldsQueryEx.java @@ -24,12 +24,12 @@ * {@link SqlFieldsQuery} with JDBC flavor - it has additional flag indicating whether JDBC driver expects * this query to return a result set or an update counter. This class is not intended for use outside JDBC driver. */ -public final class JdbcSqlFieldsQuery extends SqlFieldsQuery { +public final class SqlFieldsQueryEx extends SqlFieldsQuery { /** */ private static final long serialVersionUID = 0L; /** Flag set by JDBC driver to enforce checks for correct operation type. */ - private final boolean isQry; + private final Boolean isQry; /** Whether server side DML should be enabled. */ private boolean updateOnServer; @@ -38,7 +38,7 @@ public final class JdbcSqlFieldsQuery extends SqlFieldsQuery { * @param sql SQL query. * @param isQry Flag indicating whether this object denotes a query or an update operation. */ - public JdbcSqlFieldsQuery(String sql, boolean isQry) { + public SqlFieldsQueryEx(String sql, Boolean isQry) { super(sql); this.isQry = isQry; } @@ -46,61 +46,61 @@ public JdbcSqlFieldsQuery(String sql, boolean isQry) { /** * @return Flag indicating whether this object denotes a query or an update operation.. */ - public boolean isQuery() { + public Boolean isQuery() { return isQry; } /** {@inheritDoc} */ - @Override public JdbcSqlFieldsQuery setSql(String sql) { + @Override public SqlFieldsQueryEx setSql(String sql) { super.setSql(sql); return this; } /** {@inheritDoc} */ - @Override public JdbcSqlFieldsQuery setArgs(Object... args) { + @Override public SqlFieldsQueryEx setArgs(Object... args) { super.setArgs(args); return this; } /** {@inheritDoc} */ - @Override public JdbcSqlFieldsQuery setTimeout(int timeout, TimeUnit timeUnit) { + @Override public SqlFieldsQueryEx setTimeout(int timeout, TimeUnit timeUnit) { super.setTimeout(timeout, timeUnit); return this; } /** {@inheritDoc} */ - @Override public JdbcSqlFieldsQuery setCollocated(boolean collocated) { + @Override public SqlFieldsQueryEx setCollocated(boolean collocated) { super.setCollocated(collocated); return this; } /** {@inheritDoc} */ - @Override public JdbcSqlFieldsQuery setEnforceJoinOrder(boolean enforceJoinOrder) { + @Override public SqlFieldsQueryEx setEnforceJoinOrder(boolean enforceJoinOrder) { super.setEnforceJoinOrder(enforceJoinOrder); return this; } /** {@inheritDoc} */ - @Override public JdbcSqlFieldsQuery setDistributedJoins(boolean distributedJoins) { + @Override public SqlFieldsQueryEx setDistributedJoins(boolean distributedJoins) { super.setDistributedJoins(distributedJoins); return this; } /** {@inheritDoc} */ - @Override public JdbcSqlFieldsQuery setPageSize(int pageSize) { + @Override public SqlFieldsQueryEx setPageSize(int pageSize) { super.setPageSize(pageSize); return this; } /** {@inheritDoc} */ - @Override public JdbcSqlFieldsQuery setLocal(boolean loc) { + @Override public SqlFieldsQueryEx setLocal(boolean loc) { super.setLocal(loc); return this; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java index e25d8e24f7a6f..91d3d778dc92d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java @@ -35,7 +35,7 @@ import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteVersionUtils; import org.apache.ignite.internal.binary.BinaryWriterExImpl; -import org.apache.ignite.internal.jdbc2.JdbcSqlFieldsQuery; +import org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.cache.QueryCursorImpl; import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion; @@ -268,17 +268,17 @@ private JdbcResponse executeQuery(JdbcQueryExecuteRequest req) { break; case SELECT_STATEMENT_TYPE: - qry = new JdbcSqlFieldsQuery(sql, true); + qry = new SqlFieldsQueryEx(sql, true); break; default: assert req.expectedStatementType() == JdbcStatementType.UPDATE_STMT_TYPE; - qry = new JdbcSqlFieldsQuery(sql, false); + qry = new SqlFieldsQueryEx(sql, false); if (updateOnServer) - ((JdbcSqlFieldsQuery)qry).setUpdateOnServer(true); + ((SqlFieldsQueryEx)qry).setUpdateOnServer(true); } qry.setArgs(req.arguments()); @@ -484,7 +484,7 @@ private ClientListenerResponse executeBatch(JdbcBatchExecuteRequest req) { if (q.sql() != null) sql = q.sql(); - SqlFieldsQuery qry = new JdbcSqlFieldsQuery(sql, false); + SqlFieldsQuery qry = new SqlFieldsQueryEx(sql, false); qry.setArgs(q.args()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java index a4af478ebed2e..298bc12244420 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java @@ -37,8 +37,11 @@ public class OdbcConnectionContext implements ClientListenerConnectionContext { /** Version 2.1.5: added "lazy" flag. */ public static final ClientListenerProtocolVersion VER_2_1_5 = ClientListenerProtocolVersion.create(2, 1, 5); + /** Version 2.3.0: added "updateOnServer" flag. */ + public static final ClientListenerProtocolVersion VER_2_3_0 = ClientListenerProtocolVersion.create(2, 3, 0); + /** Current version. */ - private static final ClientListenerProtocolVersion CURRENT_VER = VER_2_1_5; + private static final ClientListenerProtocolVersion CURRENT_VER = VER_2_3_0; /** Supported versions. */ private static final Set SUPPORTED_VERS = new HashSet<>(); @@ -60,6 +63,7 @@ public class OdbcConnectionContext implements ClientListenerConnectionContext { static { SUPPORTED_VERS.add(CURRENT_VER); + SUPPORTED_VERS.add(VER_2_1_5); SUPPORTED_VERS.add(VER_2_1_0); } @@ -98,8 +102,13 @@ public OdbcConnectionContext(GridKernalContext ctx, GridSpinBusyLock busyLock, i if (ver.compareTo(VER_2_1_5) >= 0) lazy = reader.readBoolean(); + boolean updateOnServer = false; + + if (ver.compareTo(VER_2_3_0) >= 0) + updateOnServer = reader.readBoolean(); + handler = new OdbcRequestHandler(ctx, busyLock, maxCursors, distributedJoins, - enforceJoinOrder, replicatedOnly, collocated, lazy); + enforceJoinOrder, replicatedOnly, collocated, lazy, updateOnServer); parser = new OdbcMessageParser(ctx, ver); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java index 07b41f3286122..ffb9a2945b6ad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java @@ -34,16 +34,17 @@ import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.binary.BinaryWriterExImpl; import org.apache.ignite.internal.binary.GridBinaryMarshaller; +import org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.cache.QueryCursorImpl; import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; import org.apache.ignite.internal.processors.odbc.ClientListenerRequest; import org.apache.ignite.internal.processors.odbc.ClientListenerRequestHandler; import org.apache.ignite.internal.processors.odbc.ClientListenerResponse; +import org.apache.ignite.internal.processors.odbc.jdbc.JdbcStatementType; import org.apache.ignite.internal.processors.odbc.odbc.escape.OdbcEscapeUtils; import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata; import org.apache.ignite.internal.processors.query.GridQueryIndexing; import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor; -import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.util.GridSpinBusyLock; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.U; @@ -94,6 +95,9 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler { /** Lazy flag. */ private final boolean lazy; + /** Update on server flag. */ + private final boolean updateOnServer; + /** * Constructor. * @param ctx Context. @@ -104,10 +108,11 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler { * @param replicatedOnly Replicated only flag. * @param collocated Collocated flag. * @param lazy Lazy flag. + * @param updateOnServer Update on server flag. */ public OdbcRequestHandler(GridKernalContext ctx, GridSpinBusyLock busyLock, int maxCursors, boolean distributedJoins, boolean enforceJoinOrder, boolean replicatedOnly, - boolean collocated, boolean lazy) { + boolean collocated, boolean lazy, boolean updateOnServer) { this.ctx = ctx; this.busyLock = busyLock; this.maxCursors = maxCursors; @@ -116,6 +121,7 @@ public OdbcRequestHandler(GridKernalContext ctx, GridSpinBusyLock busyLock, int this.replicatedOnly = replicatedOnly; this.collocated = collocated; this.lazy = lazy; + this.updateOnServer = updateOnServer; log = ctx.log(getClass()); } @@ -196,8 +202,8 @@ public void onDisconnect() { * @param args Arguments. * @return Query instance. */ - private SqlFieldsQuery makeQuery(String schema, String sql, Object[] args) { - SqlFieldsQuery qry = new SqlFieldsQuery(sql); + private SqlFieldsQueryEx makeQuery(String schema, String sql, Object[] args) { + SqlFieldsQueryEx qry = new SqlFieldsQueryEx(sql, null); qry.setArgs(args); @@ -207,6 +213,7 @@ private SqlFieldsQuery makeQuery(String schema, String sql, Object[] args) { qry.setCollocated(collocated); qry.setLazy(lazy); qry.setSchema(schema); + qry.setUpdateOnServer(updateOnServer); return qry; } diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index 2703e6d30e340..f992739526e61 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -310,7 +310,7 @@ org.apache.ignite.internal.jdbc2.JdbcDatabaseMetadata$UpdateMetadataTask org.apache.ignite.internal.jdbc2.JdbcQueryTask org.apache.ignite.internal.jdbc2.JdbcQueryTask$1 org.apache.ignite.internal.jdbc2.JdbcQueryTask$QueryResult -org.apache.ignite.internal.jdbc2.JdbcSqlFieldsQuery +org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx org.apache.ignite.internal.managers.GridManagerAdapter$1$1 org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager$CheckpointSet org.apache.ignite.internal.managers.checkpoint.GridCheckpointRequest @@ -2094,4 +2094,4 @@ org.apache.ignite.transactions.TransactionRollbackException org.apache.ignite.transactions.TransactionState org.apache.ignite.transactions.TransactionTimeoutException org.apache.ignite.util.AttributeNodeFilter -org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIO \ No newline at end of file +org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIO diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index ef39a84cb9487..91e487cadac5c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -50,7 +50,7 @@ import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.GridKernalContext; -import org.apache.ignite.internal.jdbc2.JdbcSqlFieldsQuery; +import org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheOperationContext; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; @@ -414,7 +414,7 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo } if (plan.distributed && !loc && !fieldsQry.isLocal() && - fieldsQry instanceof JdbcSqlFieldsQuery && ((JdbcSqlFieldsQuery)fieldsQry).isUpdateOnServer()) { + fieldsQry instanceof SqlFieldsQueryEx && ((SqlFieldsQueryEx)fieldsQry).isUpdateOnServer()) { UpdateResult result = doDistributedUpdate(schemaName, fieldsQry, plan, cancel); // null is returned in case not all nodes support distributed DML. @@ -513,7 +513,7 @@ private UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared res = UpdatePlanBuilder.planForStatement(p, errKeysPos); - if (fieldsQry instanceof JdbcSqlFieldsQuery && ((JdbcSqlFieldsQuery)fieldsQry).isUpdateOnServer() + if (fieldsQry instanceof SqlFieldsQueryEx && ((SqlFieldsQueryEx)fieldsQry).isUpdateOnServer() && !loc && !F.isEmpty(res.selectQry)) checkPlanCanBeDistributed(fieldsQry, conn, res); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 4d0fecc949d9a..8028fb72e5583 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -59,7 +59,7 @@ import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.GridTopic; import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.jdbc2.JdbcSqlFieldsQuery; +import org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheEntryImpl; import org.apache.ignite.internal.processors.cache.CacheObject; @@ -1518,7 +1518,8 @@ UpdateResult runDistributedUpdate( * @param isQry {@code true} for select queries, otherwise (DML/DDL queries) {@code false}. */ private void checkQueryType(SqlFieldsQuery qry, boolean isQry) { - if (qry instanceof JdbcSqlFieldsQuery && ((JdbcSqlFieldsQuery)qry).isQuery() != isQry) + if (qry instanceof SqlFieldsQueryEx && ((SqlFieldsQueryEx)qry).isQuery() != null && + ((SqlFieldsQueryEx)qry).isQuery() != isQry) throw new IgniteSQLException("Given statement type does not match that declared by JDBC driver", IgniteQueryErrorCode.STMT_TYPE_MISMATCH); } diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java index 96708bf2ce01d..10c513cbb396e 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java @@ -13,13 +13,13 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.jdbc2.JdbcSqlFieldsQuery; +import org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; -/** Tests for {@link JdbcSqlFieldsQuery#updateOnServer} flag. */ +/** Tests for {@link SqlFieldsQueryEx#updateOnServer} flag. */ public class IgniteSqlDistributedDmlFlagSelfTest extends GridCommonAbstractTest { /** IP finder. */ private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); @@ -183,7 +183,7 @@ public void testUpdate() throws Exception { String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE depo > 0"; - checkUpdate(client.cache(CACHE_ACCOUNT), accounts, new JdbcSqlFieldsQuery(text, false).setArgs(10)); + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, new SqlFieldsQueryEx(text, false).setArgs(10)); } /** @@ -196,7 +196,7 @@ public void testUpdateFastKey() throws Exception { String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE _key = ?"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new JdbcSqlFieldsQuery(text, false).setArgs(10, 1)); + new SqlFieldsQueryEx(text, false).setArgs(10, 1)); } /** @@ -209,7 +209,7 @@ public void testUpdateLimit() throws Exception { String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE sn >= ? AND sn < ? LIMIT ?"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new JdbcSqlFieldsQuery(text, false).setArgs(10, 0, 10, 10)); + new SqlFieldsQueryEx(text, false).setArgs(10, 0, 10, 10)); } /** @@ -227,7 +227,7 @@ public void testUpdateWhereSubquery() throws Exception { "WHERE accountId IN (SELECT p._key FROM \"acc\".Account p WHERE depo < ?)"; checkUpdate(client.cache(CACHE_TRADE), trades, - new JdbcSqlFieldsQuery(text, false).setArgs(0, 0)); + new SqlFieldsQueryEx(text, false).setArgs(0, 0)); } /** @@ -244,7 +244,7 @@ public void testUpdateSetSubquery() throws Exception { "(SELECT a.depo/t.price FROM \"acc\".Account a WHERE t.accountId = a._key)"; checkUpdate(client.cache(CACHE_TRADE), trades, - new JdbcSqlFieldsQuery(text, false)); + new SqlFieldsQueryEx(text, false)); } /** @@ -261,7 +261,7 @@ public void testUpdateSetTableSubquery() throws Exception { "(SELECT a.depo/t.price FROM \"acc\".Account a WHERE t.accountId = a._key)"; checkUpdate(client.cache(CACHE_TRADE), trades, - new JdbcSqlFieldsQuery(text, false)); + new SqlFieldsQueryEx(text, false)); } /** @@ -273,7 +273,7 @@ public void testInsertValues() throws Exception { " VALUES (?, ?, ?, ?), (?, ?, ?, ?)"; checkUpdate(client.cache(CACHE_ACCOUNT), null, - new JdbcSqlFieldsQuery(text, false).setArgs(1, "John Marry", 11111, 100, 2, "Marry John", 11112, 200)); + new SqlFieldsQueryEx(text, false).setArgs(1, "John Marry", 11111, 100, 2, "Marry John", 11112, 200)); } /** @@ -289,7 +289,7 @@ public void testInsertFromSelect() throws Exception { "SELECT a._key, a._key, ?, a.depo/?, ? FROM \"acc\".Account a"; checkUpdate(client.cache(CACHE_TRADE), null, - new JdbcSqlFieldsQuery(text, false).setArgs(1, 10, 10)); + new SqlFieldsQueryEx(text, false).setArgs(1, 10, 10)); } /** @@ -306,7 +306,7 @@ public void testInsertFromSelectOrderBy() throws Exception { "ORDER BY a.sn DESC"; checkUpdate(client.cache(CACHE_TRADE), null, - new JdbcSqlFieldsQuery(text, false).setArgs(1, 10, 10)); + new SqlFieldsQueryEx(text, false).setArgs(1, 10, 10)); } /** @@ -324,7 +324,7 @@ public void testInsertFromSelectUnion() throws Exception { "SELECT 101 + a2._key, a2._key, 1, a2.depo, 1 FROM \"acc\".Account a2"; checkUpdate(client.cache(CACHE_TRADE), null, - new JdbcSqlFieldsQuery(text, false)); + new SqlFieldsQueryEx(text, false)); } /** @@ -344,7 +344,7 @@ public void testInsertFromSelectGroupBy() throws Exception { "GROUP BY accountId"; checkUpdate(client.cache(CACHE_REPORT), null, - new JdbcSqlFieldsQuery(text, false)); + new SqlFieldsQueryEx(text, false)); } /** @@ -360,7 +360,7 @@ public void testInsertFromSelectDistinct() throws Exception { "SELECT DISTINCT sn, name FROM \"acc\".Account "; checkUpdate(client.cache(CACHE_LIST), null, - new JdbcSqlFieldsQuery(text, false)); + new SqlFieldsQueryEx(text, false)); } /** @@ -379,7 +379,7 @@ public void testInsertFromSelectJoin() throws Exception { "FROM \"acc\".Account a JOIN \"stock\".Stock s ON 1=1"; checkUpdate(client.cache(CACHE_TRADE), null, - new JdbcSqlFieldsQuery(text, false).setArgs(10, 10)); + new SqlFieldsQueryEx(text, false).setArgs(10, 10)); } /** @@ -394,7 +394,7 @@ public void testDelete() throws Exception { String text = "DELETE FROM \"acc\".Account WHERE sn > ?"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new JdbcSqlFieldsQuery(text, false).setArgs(10)); + new SqlFieldsQueryEx(text, false).setArgs(10)); } /** @@ -409,7 +409,7 @@ public void testDeleteTop() throws Exception { String text = "DELETE TOP ? FROM \"acc\".Account WHERE sn < ?"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new JdbcSqlFieldsQuery(text, false).setArgs(10, 10)); + new SqlFieldsQueryEx(text, false).setArgs(10, 10)); } /** @@ -427,7 +427,7 @@ public void testDeleteWhereSubquery() throws Exception { "WHERE _key IN (SELECT t.accountId FROM \"trade\".Trade t)"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new JdbcSqlFieldsQuery(text, false)); + new SqlFieldsQueryEx(text, false)); } /** @@ -441,7 +441,7 @@ public void testMergeValues() throws Exception { " VALUES (?, ?, ?, ?), (?, ?, ?, ?)"; checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new JdbcSqlFieldsQuery(text, false).setArgs(0, "John Marry", 11111, 100, 1, "Marry John", 11112, 200)); + new SqlFieldsQueryEx(text, false).setArgs(0, "John Marry", 11111, 100, 1, "Marry John", 11112, 200)); } /** @@ -464,7 +464,7 @@ public void testMergeFromSelectJoin() throws Exception { "FROM \"acc\".Account a JOIN \"stock\".Stock s ON 1=1"; checkUpdate(client.cache(CACHE_TRADE), trades, - new JdbcSqlFieldsQuery(text, false).setArgs(10, 10)); + new SqlFieldsQueryEx(text, false).setArgs(10, 10)); } /** @@ -485,7 +485,7 @@ public void testMergeFromSelectOrderBy() throws Exception { "ORDER BY a.sn DESC"; checkUpdate(client.cache(CACHE_TRADE), trades, - new JdbcSqlFieldsQuery(text, false).setArgs(1, 10, 10)); + new SqlFieldsQueryEx(text, false).setArgs(1, 10, 10)); } /** @@ -509,7 +509,7 @@ public void testMergeFromSelectGroupBy() throws Exception { "GROUP BY accountId"; checkUpdate(client.cache(CACHE_REPORT), reports, - new JdbcSqlFieldsQuery(text, false)); + new SqlFieldsQueryEx(text, false)); } /** @@ -580,7 +580,7 @@ private Map getTrades(int numAccounts, int numStocks) { * @param Key type. * @param Value type. */ - private void checkUpdate(IgniteCache cache, Map initial, JdbcSqlFieldsQuery qry) { + private void checkUpdate(IgniteCache cache, Map initial, SqlFieldsQueryEx qry) { cache.clear(); if (!F.isEmpty(initial)) diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java index 70d3fd03c7676..0f86460540822 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java @@ -42,7 +42,7 @@ import org.apache.ignite.events.CacheQueryExecutedEvent; import org.apache.ignite.events.Event; import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.jdbc2.JdbcSqlFieldsQuery; +import org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor; import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor; @@ -210,7 +210,7 @@ public void testSimpleUpdateDistributedReplicated() throws Exception { Position p = cache.get(1); - List> r = cache.query(new JdbcSqlFieldsQuery("UPDATE Position p SET name = CONCAT('A ', name)", false) + List> r = cache.query(new SqlFieldsQueryEx("UPDATE Position p SET name = CONCAT('A ', name)", false) .setUpdateOnServer(true)).getAll(); assertEquals((long)cache.size(), r.get(0).get(0)); @@ -227,7 +227,7 @@ public void testSimpleUpdateDistributedPartitioned() throws Exception { IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); - List> r = cache.query(new JdbcSqlFieldsQuery( + List> r = cache.query(new SqlFieldsQueryEx( "UPDATE Person SET position = CASEWHEN(position = 1, 1, position - 1)", false) .setUpdateOnServer(true)).getAll(); @@ -246,7 +246,7 @@ public void testDistributedUpdateFailedKeys() throws Exception { GridTestUtils.assertThrows(log, new Callable() { @Override public Object call() { - return cache.query(new JdbcSqlFieldsQuery("UPDATE Organization SET rate = Modify(_key, rate - 1)", false) + return cache.query(new SqlFieldsQueryEx("UPDATE Organization SET rate = Modify(_key, rate - 1)", false) .setUpdateOnServer(true)); } }, CacheException.class, "Failed to update some keys because they had been modified concurrently"); @@ -263,7 +263,7 @@ public void testDistributedUpdateFail() throws Exception { GridTestUtils.assertThrows(log, new Callable() { @Override public Object call() { - return cache.query(new JdbcSqlFieldsQuery("UPDATE Person SET name = Fail(name)", false) + return cache.query(new SqlFieldsQueryEx("UPDATE Person SET name = Fail(name)", false) .setUpdateOnServer(true)); } }, CacheException.class, "Failed to execute SQL query"); @@ -286,7 +286,7 @@ public void testQueryParallelism() throws Exception { for (int i = 0; i < 1024; i++) cache.put(i, new Organization("Acme Inc #" + i, 0)); - List> r = cache.query(new JdbcSqlFieldsQuery("UPDATE \"" + cacheName + + List> r = cache.query(new SqlFieldsQueryEx("UPDATE \"" + cacheName + "\".Organization o SET name = UPPER(name)", false).setUpdateOnServer(true)).getAll(); assertEquals((long)cache.size(), r.get(0).get(0)); @@ -321,7 +321,7 @@ public void testEvents() throws Exception { for (int i = 0; i < 1024; i++) cache.put(i, new Organization("Acme Inc #" + i, 0)); - cache.query(new JdbcSqlFieldsQuery("UPDATE \"org\".Organization o SET name = UPPER(name)", false) + cache.query(new SqlFieldsQueryEx("UPDATE \"org\".Organization o SET name = UPPER(name)", false) .setUpdateOnServer(true)).getAll(); assertTrue(latch.await(5000, MILLISECONDS)); @@ -348,7 +348,7 @@ public void testSpecificPartitionsUpdate() throws Exception { IgniteCache cache = grid(NODE_CLIENT).cache(CACHE_PERSON); // UPDATE over even partitions - cache.query(new JdbcSqlFieldsQuery("UPDATE Person SET position = 0", false) + cache.query(new SqlFieldsQueryEx("UPDATE Person SET position = 0", false) .setUpdateOnServer(true) .setPartitions(parts)); @@ -376,7 +376,7 @@ public void testCancel() throws Exception { final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { @Override public Object call() { - return cache.query(new JdbcSqlFieldsQuery("UPDATE Organization SET name = WAIT(name)", false) + return cache.query(new SqlFieldsQueryEx("UPDATE Organization SET name = WAIT(name)", false) .setUpdateOnServer(true)); } }); @@ -422,7 +422,7 @@ public void testNodeStopDuringUpdate() throws Exception { final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { @Override public Object call() { - return cache.query(new JdbcSqlFieldsQuery("UPDATE Organization SET name = WAIT(name)", false) + return cache.query(new SqlFieldsQueryEx("UPDATE Organization SET name = WAIT(name)", false) .setUpdateOnServer(true)); } }); From 3c60529d56bef12c20d86356c9e81088b6bd6157 Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Tue, 10 Oct 2017 16:36:22 +0300 Subject: [PATCH 27/40] IGNITE-6024: Moved SqlFieldsQueryEx to cache processor --- .../ignite/internal/jdbc2/JdbcBatchUpdateTask.java | 1 + .../internal/jdbc2/JdbcQueryMultipleStatementsTask.java | 1 + .../org/apache/ignite/internal/jdbc2/JdbcQueryTask.java | 1 + .../cache/query}/SqlFieldsQueryEx.java | 9 ++++----- .../processors/odbc/jdbc/JdbcRequestHandler.java | 2 +- .../processors/odbc/odbc/OdbcRequestHandler.java | 3 +-- .../src/main/resources/META-INF/classnames.properties | 2 +- .../processors/query/h2/DmlStatementsProcessor.java | 2 +- .../internal/processors/query/h2/IgniteH2Indexing.java | 2 +- .../query/IgniteSqlDistributedDmlFlagSelfTest.java | 2 +- .../query/IgniteSqlDistributedDmlSelfTest.java | 2 +- 11 files changed, 14 insertions(+), 13 deletions(-) rename modules/core/src/main/java/org/apache/ignite/internal/{jdbc2 => processors/cache/query}/SqlFieldsQueryEx.java (92%) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBatchUpdateTask.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBatchUpdateTask.java index 9bf859d9cb4d2..774f9229babfc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBatchUpdateTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcBatchUpdateTask.java @@ -29,6 +29,7 @@ import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.processors.cache.QueryCursorImpl; import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; +import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.lang.IgniteCallable; import org.apache.ignite.resources.IgniteInstanceResource; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryMultipleStatementsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryMultipleStatementsTask.java index 0ff51ae3405cf..f907525571a5f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryMultipleStatementsTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryMultipleStatementsTask.java @@ -27,6 +27,7 @@ import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.processors.cache.QueryCursorImpl; +import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.lang.IgniteCallable; import org.apache.ignite.resources.IgniteInstanceResource; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java index 7e0bdddd30779..d97b833c81da6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java @@ -36,6 +36,7 @@ import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.processors.cache.QueryCursorImpl; +import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata; import org.apache.ignite.internal.util.typedef.CAX; import org.apache.ignite.internal.util.typedef.internal.U; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/SqlFieldsQueryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java similarity index 92% rename from modules/core/src/main/java/org/apache/ignite/internal/jdbc2/SqlFieldsQueryEx.java rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java index 0d003d5acdc0c..b2a495b0cd42f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/SqlFieldsQueryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java @@ -15,20 +15,19 @@ * limitations under the License. */ -package org.apache.ignite.internal.jdbc2; +package org.apache.ignite.internal.processors.cache.query; import java.util.concurrent.TimeUnit; import org.apache.ignite.cache.query.SqlFieldsQuery; /** - * {@link SqlFieldsQuery} with JDBC flavor - it has additional flag indicating whether JDBC driver expects - * this query to return a result set or an update counter. This class is not intended for use outside JDBC driver. + * {@link SqlFieldsQuery} with experimental and internal features. */ public final class SqlFieldsQueryEx extends SqlFieldsQuery { /** */ private static final long serialVersionUID = 0L; - /** Flag set by JDBC driver to enforce checks for correct operation type. */ + /** Flag to enforce checks for correct operation type. */ private final Boolean isQry; /** Whether server side DML should be enabled. */ @@ -44,7 +43,7 @@ public SqlFieldsQueryEx(String sql, Boolean isQry) { } /** - * @return Flag indicating whether this object denotes a query or an update operation.. + * @return Flag indicating whether this object denotes a query or an update operation. */ public Boolean isQuery() { return isQry; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java index 91d3d778dc92d..add9ab45a6efb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java @@ -35,7 +35,7 @@ import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteVersionUtils; import org.apache.ignite.internal.binary.BinaryWriterExImpl; -import org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx; +import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.cache.QueryCursorImpl; import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java index ffb9a2945b6ad..f4564a4610bdc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java @@ -34,13 +34,12 @@ import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.binary.BinaryWriterExImpl; import org.apache.ignite.internal.binary.GridBinaryMarshaller; -import org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx; +import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.cache.QueryCursorImpl; import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; import org.apache.ignite.internal.processors.odbc.ClientListenerRequest; import org.apache.ignite.internal.processors.odbc.ClientListenerRequestHandler; import org.apache.ignite.internal.processors.odbc.ClientListenerResponse; -import org.apache.ignite.internal.processors.odbc.jdbc.JdbcStatementType; import org.apache.ignite.internal.processors.odbc.odbc.escape.OdbcEscapeUtils; import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata; import org.apache.ignite.internal.processors.query.GridQueryIndexing; diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties index f992739526e61..2f795dfd8de0f 100644 --- a/modules/core/src/main/resources/META-INF/classnames.properties +++ b/modules/core/src/main/resources/META-INF/classnames.properties @@ -310,7 +310,7 @@ org.apache.ignite.internal.jdbc2.JdbcDatabaseMetadata$UpdateMetadataTask org.apache.ignite.internal.jdbc2.JdbcQueryTask org.apache.ignite.internal.jdbc2.JdbcQueryTask$1 org.apache.ignite.internal.jdbc2.JdbcQueryTask$QueryResult -org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx +org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx org.apache.ignite.internal.managers.GridManagerAdapter$1$1 org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager$CheckpointSet org.apache.ignite.internal.managers.checkpoint.GridCheckpointRequest diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 91e487cadac5c..1da3a8df9782e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -50,7 +50,7 @@ import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.GridKernalContext; -import org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx; +import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheOperationContext; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index 8028fb72e5583..c8b45ca2d92c4 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -59,7 +59,7 @@ import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.GridTopic; import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx; +import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheEntryImpl; import org.apache.ignite.internal.processors.cache.CacheObject; diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java index 10c513cbb396e..3d00e3afc99d7 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java @@ -13,7 +13,7 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx; +import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java index 0f86460540822..1113dd511cacb 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java @@ -42,7 +42,7 @@ import org.apache.ignite.events.CacheQueryExecutedEvent; import org.apache.ignite.events.Event; import org.apache.ignite.internal.IgniteInternalFuture; -import org.apache.ignite.internal.jdbc2.SqlFieldsQueryEx; +import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor; import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor; From c28566d92fff8b6ee11bb73e7dc97be0b3bcf86f Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Tue, 10 Oct 2017 16:54:42 +0300 Subject: [PATCH 28/40] IGNITE-6024: fixed INSERT VALUES case and broken tests --- .../processors/cache/query/SqlFieldsQueryEx.java | 16 ++++++++++++++++ .../query/h2/DmlStatementsProcessor.java | 2 +- .../processors/query/h2/IgniteH2Indexing.java | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java index b2a495b0cd42f..9c77063948303 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java @@ -42,6 +42,22 @@ public SqlFieldsQueryEx(String sql, Boolean isQry) { this.isQry = isQry; } + /** + * @param qry SQL query. + */ + public SqlFieldsQueryEx(SqlFieldsQuery qry) { + super(qry); + + if (qry instanceof SqlFieldsQueryEx) { + SqlFieldsQueryEx other = (SqlFieldsQueryEx)qry; + + this.isQry = other.isQry; + this.updateOnServer = other.updateOnServer; + } + else + this.isQry = null; + } + /** * @return Flag indicating whether this object denotes a query or an update operation. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 1da3a8df9782e..540e92096c7df 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -514,7 +514,7 @@ private UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared res = UpdatePlanBuilder.planForStatement(p, errKeysPos); if (fieldsQry instanceof SqlFieldsQueryEx && ((SqlFieldsQueryEx)fieldsQry).isUpdateOnServer() - && !loc && !F.isEmpty(res.selectQry)) + && !loc && res.rowsNum == 0 && !F.isEmpty(res.selectQry)) checkPlanCanBeDistributed(fieldsQry, conn, res); // Don't cache re-runs diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index c8b45ca2d92c4..fe2dbcd20e3a0 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -1451,7 +1451,7 @@ UpdateResult runDistributedUpdate( if (DmlStatementsProcessor.isDmlStatement(prepared)) { try { res.add(dmlProc.updateSqlFieldsDistributed(schemaName, c, prepared, - new SqlFieldsQuery(qry).setSql(sqlQry).setArgs(args), cancel)); + new SqlFieldsQueryEx(qry).setSql(sqlQry).setArgs(args), cancel)); continue; } From 45f16b52f0815c462e4543a43cc2660df9f41050 Mon Sep 17 00:00:00 2001 From: Igor Sapego Date: Tue, 10 Oct 2017 17:54:28 +0300 Subject: [PATCH 29/40] IGNITE-6024: Implemented C++ part --- .../cpp/odbc-test/src/configuration_test.cpp | 25 +++++++++++++----- .../cpp/odbc-test/src/queries_test.cpp | 8 ++++++ .../ignite/odbc/config/configuration.h | 26 +++++++++++++++++++ .../cpp/odbc/include/ignite/odbc/message.h | 6 ++++- .../include/ignite/odbc/protocol_version.h | 1 + .../odbc/system/ui/dsn_configuration_window.h | 4 +++ .../system/ui/dsn_configuration_window.cpp | 17 ++++++++++++ .../cpp/odbc/src/config/configuration.cpp | 2 ++ modules/platforms/cpp/odbc/src/connection.cpp | 5 +++- modules/platforms/cpp/odbc/src/dsn_config.cpp | 3 +++ modules/platforms/cpp/odbc/src/message.cpp | 12 ++++++--- .../cpp/odbc/src/protocol_version.cpp | 6 +++-- 12 files changed, 102 insertions(+), 13 deletions(-) diff --git a/modules/platforms/cpp/odbc-test/src/configuration_test.cpp b/modules/platforms/cpp/odbc-test/src/configuration_test.cpp index 7da67576b8474..3a331144bd282 100644 --- a/modules/platforms/cpp/odbc-test/src/configuration_test.cpp +++ b/modules/platforms/cpp/odbc-test/src/configuration_test.cpp @@ -43,6 +43,7 @@ namespace const bool testReplicatedOnly = true; const bool testCollocated = true; const bool testLazy = true; + const bool testUpdateOnServer = true; const std::string testAddress = testServerHost + ':' + ignite::common::LexicalCast(testServerPort); } @@ -132,6 +133,7 @@ void CheckConnectionConfig(const Configuration& cfg) BOOST_CHECK_EQUAL(cfg.IsReplicatedOnly(), testReplicatedOnly); BOOST_CHECK_EQUAL(cfg.IsCollocated(), testCollocated); BOOST_CHECK_EQUAL(cfg.IsLazy(), testLazy); + BOOST_CHECK_EQUAL(cfg.IsUpdateOnServer(), testUpdateOnServer); std::stringstream constructor; @@ -143,7 +145,8 @@ void CheckConnectionConfig(const Configuration& cfg) << "lazy=" << BoolToStr(testLazy) << ';' << "page_size=" << testPageSize << ';' << "replicated_only=" << BoolToStr(testReplicatedOnly) << ';' - << "schema=" << testSchemaName << ';'; + << "schema=" << testSchemaName << ';' + << "update_on_server=" << BoolToStr(testReplicatedOnly) << ';'; const std::string& expectedStr = constructor.str(); @@ -164,6 +167,7 @@ void CheckDsnConfig(const Configuration& cfg) BOOST_CHECK_EQUAL(cfg.IsReplicatedOnly(), false); BOOST_CHECK_EQUAL(cfg.IsCollocated(), false); BOOST_CHECK_EQUAL(cfg.IsLazy(), false); + BOOST_CHECK_EQUAL(cfg.IsUpdateOnServer(), false); } BOOST_AUTO_TEST_SUITE(ConfigurationTestSuite) @@ -180,6 +184,8 @@ BOOST_AUTO_TEST_CASE(CheckTestValuesNotEquealDefault) BOOST_CHECK_NE(testEnforceJoinOrder, Configuration::DefaultValue::enforceJoinOrder); BOOST_CHECK_NE(testReplicatedOnly, Configuration::DefaultValue::replicatedOnly); BOOST_CHECK_NE(testCollocated, Configuration::DefaultValue::collocated); + BOOST_CHECK_NE(testLazy, Configuration::DefaultValue::lazy); + BOOST_CHECK_NE(testUpdateOnServer, Configuration::DefaultValue::updateOnServer); } BOOST_AUTO_TEST_CASE(TestConnectStringUppercase) @@ -196,7 +202,8 @@ BOOST_AUTO_TEST_CASE(TestConnectStringUppercase) << "COLLOCATED=" << BoolToStr(testCollocated, false) << ';' << "REPLICATED_ONLY=" << BoolToStr(testReplicatedOnly, false) << ';' << "PAGE_SIZE=" << testPageSize << ';' - << "SCHEMA=" << testSchemaName; + << "SCHEMA=" << testSchemaName << ';' + << "UPDATE_ON_SERVER=" << BoolToStr(testUpdateOnServer, false); const std::string& connectStr = constructor.str(); @@ -219,7 +226,8 @@ BOOST_AUTO_TEST_CASE(TestConnectStringLowercase) << "enforce_join_order=" << BoolToStr(testEnforceJoinOrder) << ';' << "replicated_only=" << BoolToStr(testReplicatedOnly) << ';' << "collocated=" << BoolToStr(testCollocated) << ';' - << "schema=" << testSchemaName; + << "schema=" << testSchemaName << ';' + << "update_on_server=" << BoolToStr(testUpdateOnServer); const std::string& connectStr = constructor.str(); @@ -242,7 +250,8 @@ BOOST_AUTO_TEST_CASE(TestConnectStringZeroTerminated) << "collocated=" << BoolToStr(testCollocated) << ';' << "distributed_joins=" << BoolToStr(testDistributedJoins) << ';' << "enforce_join_order=" << BoolToStr(testEnforceJoinOrder) << ';' - << "schema=" << testSchemaName; + << "schema=" << testSchemaName << ';' + << "update_on_server=" << BoolToStr(testUpdateOnServer); const std::string& connectStr = constructor.str(); @@ -265,7 +274,8 @@ BOOST_AUTO_TEST_CASE(TestConnectStringMixed) << "Enforce_Join_Order=" << BoolToStr(testEnforceJoinOrder) << ';' << "Replicated_Only=" << BoolToStr(testReplicatedOnly, false) << ';' << "Collocated=" << BoolToStr(testCollocated) << ';' - << "Schema=" << testSchemaName; + << "Schema=" << testSchemaName << ';' + << "Update_On_Server=" << BoolToStr(testUpdateOnServer); const std::string& connectStr = constructor.str(); @@ -288,7 +298,8 @@ BOOST_AUTO_TEST_CASE(TestConnectStringWhitepaces) << "COLLOCATED =" << BoolToStr(testCollocated, false) << " ;" << " REPLICATED_ONLY= " << BoolToStr(testReplicatedOnly, false) << ';' << "ENFORCE_JOIN_ORDER= " << BoolToStr(testEnforceJoinOrder, false) << " ;" - << "SCHEMA = \n\r" << testSchemaName; + << "SCHEMA = \n\r" << testSchemaName << ';' + << " update_on_server=" << BoolToStr(testUpdateOnServer, false); const std::string& connectStr = constructor.str(); @@ -358,6 +369,7 @@ BOOST_AUTO_TEST_CASE(TestConnectStringInvalidBoolKeys) keys.insert("replicated_only"); keys.insert("collocated"); keys.insert("lazy"); + keys.insert("update_on_server"); for (Set::const_iterator it = keys.begin(); it != keys.end(); ++it) { @@ -385,6 +397,7 @@ BOOST_AUTO_TEST_CASE(TestConnectStringValidBoolKeys) keys.insert("replicated_only"); keys.insert("collocated"); keys.insert("lazy"); + keys.insert("update_on_server"); for (Set::const_iterator it = keys.begin(); it != keys.end(); ++it) { diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp index 4c7e402134b0a..707669d398b2e 100644 --- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp +++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp @@ -755,6 +755,14 @@ BOOST_AUTO_TEST_CASE(TestConnectionProtocolVersion_2_1_5) InsertTestBatch(11, 20, 9); } +BOOST_AUTO_TEST_CASE(TestConnectionProtocolVersion_2_3_0) +{ + Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache;PROTOCOL_VERSION=2.3.0"); + + InsertTestStrings(10, false); + InsertTestBatch(11, 20, 9); +} + BOOST_AUTO_TEST_CASE(TestTwoRowsInt8) { CheckTwoRowsInt(SQL_C_STINYINT); diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/config/configuration.h b/modules/platforms/cpp/odbc/include/ignite/odbc/config/configuration.h index 2b1ec523aa00f..d675f82076aad 100644 --- a/modules/platforms/cpp/odbc/include/ignite/odbc/config/configuration.h +++ b/modules/platforms/cpp/odbc/include/ignite/odbc/config/configuration.h @@ -82,6 +82,9 @@ namespace ignite /** Connection attribute keyword for lazy attribute. */ static const std::string lazy; + + /** Connection attribute keyword for updateOnServer attribute. */ + static const std::string updateOnServer; }; /** Default values for configuration. */ @@ -125,6 +128,9 @@ namespace ignite /** Default value for lazy attribute. */ static const bool lazy; + + /** Default value for updateOnServer attribute. */ + static const bool updateOnServer; }; /** @@ -383,6 +389,26 @@ namespace ignite SetBoolValue(Key::lazy, val); } + /** + * Check update on server flag. + * + * @return True if update on server. + */ + bool IsUpdateOnServer() const + { + return GetBoolValue(Key::updateOnServer, DefaultValue::updateOnServer); + } + + /** + * Set update on server. + * + * @param val Value to set. + */ + void SetUpdateOnServer(bool val) + { + SetBoolValue(Key::updateOnServer, val); + } + /** * Get protocol version. * diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h index 91a808cab0f55..d88a16a7817ff 100644 --- a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h +++ b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h @@ -79,9 +79,10 @@ namespace ignite * @param replicatedOnly Replicated only flag. * @param collocated Collocated flag. * @param lazy Lazy flag. + * @param updateOnServer Update on server. */ HandshakeRequest(const ProtocolVersion& version, bool distributedJoins, bool enforceJoinOrder, - bool replicatedOnly, bool collocated, bool lazy); + bool replicatedOnly, bool collocated, bool lazy, bool updateOnServer); /** * Destructor. @@ -112,6 +113,9 @@ namespace ignite /** Lazy flag. */ bool lazy; + + /** Update on server flag. */ + bool updateOnServer; }; /** diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/protocol_version.h b/modules/platforms/cpp/odbc/include/ignite/odbc/protocol_version.h index c36d5dd709884..e6088a74b46d3 100644 --- a/modules/platforms/cpp/odbc/include/ignite/odbc/protocol_version.h +++ b/modules/platforms/cpp/odbc/include/ignite/odbc/protocol_version.h @@ -34,6 +34,7 @@ namespace ignite /** Current protocol version. */ static const ProtocolVersion VERSION_2_1_0; static const ProtocolVersion VERSION_2_1_5; + static const ProtocolVersion VERSION_2_3_0; typedef std::set VersionSet; diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/system/ui/dsn_configuration_window.h b/modules/platforms/cpp/odbc/include/ignite/odbc/system/ui/dsn_configuration_window.h index 2974b677f2c02..7b9be8718aa80 100644 --- a/modules/platforms/cpp/odbc/include/ignite/odbc/system/ui/dsn_configuration_window.h +++ b/modules/platforms/cpp/odbc/include/ignite/odbc/system/ui/dsn_configuration_window.h @@ -55,6 +55,7 @@ namespace ignite REPLICATED_ONLY_CHECK_BOX, COLLOCATED_CHECK_BOX, LAZY_CHECK_BOX, + UPDATE_ON_SERVER_CHECK_BOX, PROTOCOL_VERSION_LABEL, PROTOCOL_VERSION_COMBO_BOX, OK_BUTTON, @@ -149,6 +150,9 @@ namespace ignite /** Lazy CheckBox. */ std::auto_ptr lazyCheckBox; + /** Update on server CheckBox. */ + std::auto_ptr updateOnServerCheckBox; + /** Protocol version edit field. */ std::auto_ptr protocolVersionLabel; diff --git a/modules/platforms/cpp/odbc/os/win/src/system/ui/dsn_configuration_window.cpp b/modules/platforms/cpp/odbc/os/win/src/system/ui/dsn_configuration_window.cpp index 9b13481815d47..40f7b5ffe74dc 100644 --- a/modules/platforms/cpp/odbc/os/win/src/system/ui/dsn_configuration_window.cpp +++ b/modules/platforms/cpp/odbc/os/win/src/system/ui/dsn_configuration_window.cpp @@ -180,6 +180,11 @@ namespace ignite lazyCheckBox->SetEnabled(version >= ProtocolVersion::VERSION_2_1_5); + updateOnServerCheckBox = CreateCheckBox(editPosX + checkBoxSize + interval, rowPos, checkBoxSize, + rowSize, "Update on server", ChildId::UPDATE_ON_SERVER_CHECK_BOX, config.IsUpdateOnServer()); + + updateOnServerCheckBox->SetEnabled(version >= ProtocolVersion::VERSION_2_3_0); + rowPos += interval * 2 + rowSize; connectionSettingsGroupBox = CreateGroupBox(margin, sectionBegin, width - 2 * margin, @@ -264,6 +269,13 @@ namespace ignite break; } + case ChildId::UPDATE_ON_SERVER_CHECK_BOX: + { + updateOnServerCheckBox->SetChecked(!updateOnServerCheckBox->IsChecked()); + + break; + } + case ChildId::PROTOCOL_VERSION_COMBO_BOX: { std::string versionStr; @@ -271,6 +283,7 @@ namespace ignite ProtocolVersion version = ProtocolVersion::FromString(versionStr); lazyCheckBox->SetEnabled(version >= ProtocolVersion::VERSION_2_1_5); + updateOnServerCheckBox->SetEnabled(version >= ProtocolVersion::VERSION_2_3_0); break; } @@ -309,6 +322,7 @@ namespace ignite bool replicatedOnly; bool collocated; bool lazy; + bool updateOnServer; nameEdit->GetText(dsn); addressEdit->GetText(address); @@ -329,6 +343,7 @@ namespace ignite replicatedOnly = replicatedOnlyCheckBox->IsEnabled() && replicatedOnlyCheckBox->IsChecked(); collocated = collocatedCheckBox->IsEnabled() && collocatedCheckBox->IsChecked(); lazy = lazyCheckBox->IsEnabled() && lazyCheckBox->IsChecked(); + updateOnServer = updateOnServerCheckBox->IsEnabled() && updateOnServerCheckBox->IsChecked(); LOG_MSG("Retriving arguments:"); LOG_MSG("DSN: " << dsn); @@ -341,6 +356,7 @@ namespace ignite LOG_MSG("Replicated only: " << (replicatedOnly ? "true" : "false")); LOG_MSG("Collocated: " << (collocated ? "true" : "false")); LOG_MSG("Lazy: " << (lazy ? "true" : "false")); + LOG_MSG("Update on server: " << (updateOnServer ? "true" : "false")); if (dsn.empty()) throw IgniteError(IgniteError::IGNITE_ERR_GENERIC, "DSN name can not be empty."); @@ -355,6 +371,7 @@ namespace ignite cfg.SetReplicatedOnly(replicatedOnly); cfg.SetCollocated(collocated); cfg.SetLazy(lazy); + cfg.SetUpdateOnServer(updateOnServer); } } } diff --git a/modules/platforms/cpp/odbc/src/config/configuration.cpp b/modules/platforms/cpp/odbc/src/config/configuration.cpp index 95ed96401c9b0..6bb1ab6b95c6f 100644 --- a/modules/platforms/cpp/odbc/src/config/configuration.cpp +++ b/modules/platforms/cpp/odbc/src/config/configuration.cpp @@ -45,6 +45,7 @@ namespace ignite const std::string Configuration::Key::replicatedOnly = "replicated_only"; const std::string Configuration::Key::collocated = "collocated"; const std::string Configuration::Key::lazy = "lazy"; + const std::string Configuration::Key::updateOnServer = "update_on_server"; const std::string Configuration::DefaultValue::dsn = "Apache Ignite DSN"; const std::string Configuration::DefaultValue::driver = "Apache Ignite"; @@ -60,6 +61,7 @@ namespace ignite const bool Configuration::DefaultValue::replicatedOnly = false; const bool Configuration::DefaultValue::collocated = false; const bool Configuration::DefaultValue::lazy = false; + const bool Configuration::DefaultValue::updateOnServer = false; const ProtocolVersion& Configuration::DefaultValue::protocolVersion = ProtocolVersion::GetCurrent(); diff --git a/modules/platforms/cpp/odbc/src/connection.cpp b/modules/platforms/cpp/odbc/src/connection.cpp index 161e1c4acca1b..ecff413041c56 100644 --- a/modules/platforms/cpp/odbc/src/connection.cpp +++ b/modules/platforms/cpp/odbc/src/connection.cpp @@ -417,6 +417,7 @@ namespace ignite bool replicatedOnly = false; bool collocated = false; bool lazy = false; + bool updateOnServer = false; ProtocolVersion protocolVersion; try @@ -427,6 +428,7 @@ namespace ignite replicatedOnly = config.IsReplicatedOnly(); collocated = config.IsCollocated(); lazy = config.IsLazy(); + updateOnServer = config.IsUpdateOnServer(); } catch (const IgniteError& err) { @@ -443,7 +445,8 @@ namespace ignite return SqlResult::AI_ERROR; } - HandshakeRequest req(protocolVersion, distributedJoins, enforceJoinOrder, replicatedOnly, collocated, lazy); + HandshakeRequest req(protocolVersion, distributedJoins, enforceJoinOrder, replicatedOnly, collocated, lazy, + updateOnServer); HandshakeResponse rsp; try diff --git a/modules/platforms/cpp/odbc/src/dsn_config.cpp b/modules/platforms/cpp/odbc/src/dsn_config.cpp index c91cd8cac7ce0..96812a1350ea0 100644 --- a/modules/platforms/cpp/odbc/src/dsn_config.cpp +++ b/modules/platforms/cpp/odbc/src/dsn_config.cpp @@ -108,6 +108,8 @@ namespace ignite bool lazy = ReadDsnBool(dsn, Configuration::Key::lazy, config.IsLazy()); + bool updateOnServer = ReadDsnBool(dsn, Configuration::Key::updateOnServer, config.IsUpdateOnServer()); + std::string version = ReadDsnString(dsn, Configuration::Key::protocolVersion, config.GetProtocolVersion().ToString().c_str()); @@ -125,6 +127,7 @@ namespace ignite config.SetReplicatedOnly(replicatedOnly); config.SetCollocated(collocated); config.SetLazy(lazy); + config.SetUpdateOnServer(updateOnServer); config.SetProtocolVersion(version); config.SetPageSize(pageSize); } diff --git a/modules/platforms/cpp/odbc/src/message.cpp b/modules/platforms/cpp/odbc/src/message.cpp index 36015913dae1c..fbdfc376bce5b 100644 --- a/modules/platforms/cpp/odbc/src/message.cpp +++ b/modules/platforms/cpp/odbc/src/message.cpp @@ -23,13 +23,14 @@ namespace ignite namespace odbc { HandshakeRequest::HandshakeRequest(const ProtocolVersion& version, bool distributedJoins, - bool enforceJoinOrder, bool replicatedOnly, bool collocated, bool lazy): + bool enforceJoinOrder, bool replicatedOnly, bool collocated, bool lazy, bool updateOnServer): version(version), distributedJoins(distributedJoins), enforceJoinOrder(enforceJoinOrder), replicatedOnly(replicatedOnly), collocated(collocated), - lazy(lazy) + lazy(lazy), + updateOnServer(updateOnServer) { // No-op. } @@ -53,7 +54,12 @@ namespace ignite writer.WriteBool(enforceJoinOrder); writer.WriteBool(replicatedOnly); writer.WriteBool(collocated); - writer.WriteBool(lazy); + + if (version >= ProtocolVersion::VERSION_2_1_5) + writer.WriteBool(lazy); + + if (version >= ProtocolVersion::VERSION_2_3_0) + writer.WriteBool(updateOnServer); } QueryExecuteRequest::QueryExecuteRequest(const std::string& schema, const std::string& sql, diff --git a/modules/platforms/cpp/odbc/src/protocol_version.cpp b/modules/platforms/cpp/odbc/src/protocol_version.cpp index b668fb8d99e8f..b0b91219f9f8f 100644 --- a/modules/platforms/cpp/odbc/src/protocol_version.cpp +++ b/modules/platforms/cpp/odbc/src/protocol_version.cpp @@ -28,10 +28,12 @@ namespace ignite { const ProtocolVersion ProtocolVersion::VERSION_2_1_0(2, 1, 0); const ProtocolVersion ProtocolVersion::VERSION_2_1_5(2, 1, 5); + const ProtocolVersion ProtocolVersion::VERSION_2_3_0(2, 3, 0); ProtocolVersion::VersionSet::value_type supportedArray[] = { ProtocolVersion::VERSION_2_1_0, - ProtocolVersion::VERSION_2_1_5 + ProtocolVersion::VERSION_2_1_5, + ProtocolVersion::VERSION_2_3_0, }; const ProtocolVersion::VersionSet ProtocolVersion::supported(supportedArray, @@ -60,7 +62,7 @@ namespace ignite const ProtocolVersion& ProtocolVersion::GetCurrent() { - return VERSION_2_1_5; + return VERSION_2_3_0; } void ThrowParseError() From 7d05f373ec7fb067c845f974c617a046b2acbce8 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Tue, 10 Oct 2017 20:32:46 +0300 Subject: [PATCH 30/40] IGNITE-6024: Returned immutability for UpdatePlan --- .../query/h2/DmlStatementsProcessor.java | 68 +++--------- .../processors/query/h2/H2DmlPlanKey.java | 20 +++- .../processors/query/h2/IgniteH2Indexing.java | 4 +- .../processors/query/h2/dml/UpdatePlan.java | 68 +++++++++--- .../query/h2/dml/UpdatePlanBuilder.java | 105 ++++++++++++++++-- .../h2/twostep/GridMapQueryExecutor.java | 4 +- 6 files changed, 179 insertions(+), 90 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 540e92096c7df..b61c80a043da3 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -56,7 +56,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.QueryCursorImpl; -import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery; import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; import org.apache.ignite.internal.processors.odbc.SqlStateCode; import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator; @@ -73,7 +72,6 @@ import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlanBuilder; import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser; -import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter; import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap; import org.apache.ignite.internal.util.lang.IgniteSingletonIterator; import org.apache.ignite.internal.util.typedef.F; @@ -164,7 +162,7 @@ public void onCacheStop(String cacheName) { * @throws IgniteCheckedException if failed. */ private UpdateResult updateSqlFields(String schemaName, Connection conn, Prepared prepared, - SqlFieldsQuery fieldsQry, boolean loc, IndexingQueryFilter filters, GridQueryCancel cancel) + SqlFieldsQueryEx fieldsQry, boolean loc, IndexingQueryFilter filters, GridQueryCancel cancel) throws IgniteCheckedException { Object[] errKeys = null; @@ -217,43 +215,6 @@ else if (items == 0L) return new UpdateResult(items, errKeys); } - /** - * Checks whether the given update plan can be distributed and updates it accordingly. - * - * @param fieldsQry Initial update query. - * @param conn Connection. - * @param plan Update plan. - * @throws IgniteCheckedException if failed. - */ - private void checkPlanCanBeDistributed(SqlFieldsQuery fieldsQry, Connection conn, UpdatePlan plan) - throws IgniteCheckedException { - try { - // Get a new prepared statement for derived select query. - try (PreparedStatement stmt = conn.prepareStatement(plan.selectQry)) { - idx.bindParameters(stmt, F.asList(fieldsQry.getArgs())); - - GridCacheTwoStepQuery qry = GridSqlQuerySplitter.split(conn, - GridSqlQueryParser.prepared(stmt), - fieldsQry.getArgs(), - fieldsQry.isCollocated(), - fieldsQry.isDistributedJoins(), - fieldsQry.isEnforceJoinOrder(), idx); - - plan.distributed = qry.skipMergeTable() && - qry.mapQueries().size() == 1 && - !qry.mapQueries().get(0).hasSubQueries(); - - if (plan.distributed) { - plan.cacheIds = idx.collectCacheIds(CU.cacheId(plan.tbl.cacheName()), qry); - plan.isReplicatedOnly = qry.isReplicatedOnly(); - } - } - } - catch (SQLException e) { - throw new IgniteCheckedException(e); - } - } - /** * @param schemaName Schema. * @param c Connection. @@ -265,7 +226,7 @@ private void checkPlanCanBeDistributed(SqlFieldsQuery fieldsQry, Connection conn */ @SuppressWarnings("unchecked") QueryCursorImpl> updateSqlFieldsDistributed(String schemaName, Connection c, Prepared p, - SqlFieldsQuery fieldsQry, GridQueryCancel cancel) throws IgniteCheckedException { + SqlFieldsQueryEx fieldsQry, GridQueryCancel cancel) throws IgniteCheckedException { UpdateResult res = updateSqlFields(schemaName, c, p, fieldsQry, false, null, cancel); checkUpdateResult(res); @@ -292,7 +253,7 @@ QueryCursorImpl> updateSqlFieldsDistributed(String schemaName, Connectio */ @SuppressWarnings("unchecked") GridQueryFieldsResult updateSqlFieldsLocal(String schemaName, Connection conn, PreparedStatement stmt, - SqlFieldsQuery fieldsQry, IndexingQueryFilter filters, GridQueryCancel cancel) + SqlFieldsQueryEx fieldsQry, IndexingQueryFilter filters, GridQueryCancel cancel) throws IgniteCheckedException { UpdateResult res = updateSqlFields(schemaName, conn, GridSqlQueryParser.prepared(stmt), fieldsQry, true, filters, cancel); @@ -319,7 +280,7 @@ long streamUpdateQuery(IgniteDataStreamer streamer, PreparedStatement stmt, Obje assert p != null; - UpdatePlan plan = UpdatePlanBuilder.planForStatement(p, null); + UpdatePlan plan = UpdatePlanBuilder.planForStatement(p, true, idx, null, null, null); if (!F.eq(streamer.cacheName(), plan.tbl.rowDescriptor().context().name())) throw new IgniteSQLException("Cross cache streaming is not supported, please specify cache explicitly" + @@ -399,7 +360,7 @@ long streamUpdateQuery(IgniteDataStreamer streamer, PreparedStatement stmt, Obje */ @SuppressWarnings({"ConstantConditions", "unchecked"}) private UpdateResult executeUpdateStatement(String schemaName, final GridCacheContext cctx, Connection c, - Prepared prepared, SqlFieldsQuery fieldsQry, boolean loc, IndexingQueryFilter filters, + Prepared prepared, SqlFieldsQueryEx fieldsQry, boolean loc, IndexingQueryFilter filters, GridQueryCancel cancel, Object[] failedKeys) throws IgniteCheckedException { int mainCacheId = CU.cacheId(cctx.name()); @@ -413,8 +374,7 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo return doFastUpdate(plan, fieldsQry.getArgs()); } - if (plan.distributed && !loc && !fieldsQry.isLocal() && - fieldsQry instanceof SqlFieldsQueryEx && ((SqlFieldsQueryEx)fieldsQry).isUpdateOnServer()) { + if (plan.distributed != null) { UpdateResult result = doDistributedUpdate(schemaName, fieldsQry, plan, cancel); // null is returned in case not all nodes support distributed DML. @@ -502,20 +462,16 @@ private UpdateResult processDmlSelectResult(GridCacheContext cctx, UpdatePlan pl * @return Update plan. */ @SuppressWarnings({"unchecked", "ConstantConditions"}) - private UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared p, SqlFieldsQuery fieldsQry, + private UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared p, SqlFieldsQueryEx fieldsQry, boolean loc, @Nullable Integer errKeysPos) throws IgniteCheckedException { - H2DmlPlanKey planKey = new H2DmlPlanKey(schema, p.getSQL()); + H2DmlPlanKey planKey = new H2DmlPlanKey(schema, p.getSQL(), loc, fieldsQry); UpdatePlan res = (errKeysPos == null ? planCache.get(planKey) : null); if (res != null) return res; - res = UpdatePlanBuilder.planForStatement(p, errKeysPos); - - if (fieldsQry instanceof SqlFieldsQueryEx && ((SqlFieldsQueryEx)fieldsQry).isUpdateOnServer() - && !loc && res.rowsNum == 0 && !F.isEmpty(res.selectQry)) - checkPlanCanBeDistributed(fieldsQry, conn, res); + res = UpdatePlanBuilder.planForStatement(p, loc, idx, conn, fieldsQry, errKeysPos); // Don't cache re-runs if (errKeysPos == null) @@ -574,11 +530,13 @@ private static UpdateResult doFastUpdate(UpdatePlan plan, Object[] args) throws */ private UpdateResult doDistributedUpdate(String schemaName, SqlFieldsQuery fieldsQry, UpdatePlan plan, GridQueryCancel cancel) throws IgniteCheckedException { + assert plan.distributed != null; if (cancel == null) cancel = new GridQueryCancel(); - return idx.runDistributedUpdate(schemaName, fieldsQry, plan.cacheIds, plan.isReplicatedOnly, cancel); + return idx.runDistributedUpdate(schemaName, fieldsQry, plan.distributed.getCacheIds(), + plan.distributed.isReplicatedOnly(), cancel); } /** @@ -1087,7 +1045,7 @@ private static PageProcessingResult processPage(GridCacheContext cctx, * @return Update result. * @throws IgniteCheckedException if failed. */ - UpdateResult mapDistributedUpdate(String schemaName, PreparedStatement stmt, SqlFieldsQuery fldsQry, + UpdateResult mapDistributedUpdate(String schemaName, PreparedStatement stmt, SqlFieldsQueryEx fldsQry, IndexingQueryFilter filter, GridQueryCancel cancel, boolean local) throws IgniteCheckedException { Connection c; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java index 3a43ea1951639..ea957869f795e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.query.h2; +import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; @@ -30,20 +31,33 @@ public class H2DmlPlanKey { /** SQL. */ private final String sql; + /** Flags. */ + private final byte flags; + /** * Constructor. * * @param schemaName Schema name. * @param sql SQL. */ - public H2DmlPlanKey(String schemaName, String sql) { + public H2DmlPlanKey(String schemaName, String sql, boolean loc, SqlFieldsQueryEx fieldsQry) { this.schemaName = schemaName; this.sql = sql; + + if (!fieldsQry.isUpdateOnServer() || loc || fieldsQry.isLocal()) + this.flags = 0; // flags only relevant for server side updates. + else { + this.flags = (byte)(1 + + (fieldsQry.isDistributedJoins() ? 2 : 0) + + (fieldsQry.isEnforceJoinOrder() ? 4 : 0) + + (fieldsQry.isCollocated() ? 8 : 0)); + } } /** {@inheritDoc} */ @Override public int hashCode() { - return 31 * (schemaName != null ? schemaName.hashCode() : 0) + (sql != null ? sql.hashCode() : 0); + return 31 * (31 * (schemaName != null ? schemaName.hashCode() : 0) + (sql != null ? sql.hashCode() : 0)) + + flags; } /** {@inheritDoc} */ @@ -56,7 +70,7 @@ public H2DmlPlanKey(String schemaName, String sql) { H2DmlPlanKey other = (H2DmlPlanKey)o; - return F.eq(sql, other.sql) && F.eq(schemaName, other.schemaName); + return F.eq(sql, other.sql) && F.eq(schemaName, other.schemaName) && flags == other.flags; } /** {@inheritDoc} */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index fe2dbcd20e3a0..e05fb40695835 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -826,7 +826,7 @@ public GridQueryFieldsResult queryLocalSqlFields(final String schemaName, final Prepared p = GridSqlQueryParser.prepared(stmt); if (DmlStatementsProcessor.isDmlStatement(p)) { - SqlFieldsQuery fldsQry = new SqlFieldsQuery(qry); + SqlFieldsQueryEx fldsQry = new SqlFieldsQueryEx(qry, false); if (params != null) fldsQry.setArgs(params.toArray()); @@ -1580,7 +1580,7 @@ private FieldsQueryCursor> executeTwoStepsQuery(String schemaName, int p * @return Update result. * @throws IgniteCheckedException if failed. */ - public UpdateResult mapDistributedUpdate(String schemaName, SqlFieldsQuery fldsQry, IndexingQueryFilter filter, + public UpdateResult mapDistributedUpdate(String schemaName, SqlFieldsQueryEx fldsQry, IndexingQueryFilter filter, GridQueryCancel cancel, boolean local) throws IgniteCheckedException { Connection conn = connectionForSchema(schemaName); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 028df96e333bb..d09048e7c32eb 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -65,19 +65,13 @@ public final class UpdatePlan { /** Arguments for fast UPDATE or DELETE. */ public final FastUpdateArguments fastUpdateArgs; - /** Distributed update flag. */ - public boolean distributed = false; - - /** Integer cache identifiers. */ - public List cacheIds; - - /** Flag that query contains only replicated tables. */ - public boolean isReplicatedOnly; + /** Additional info for distributed update. */ + public final DistributedPlanInfo distributed; /** */ private UpdatePlan(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry, - int rowsNum, FastUpdateArguments fastUpdateArgs) { + int rowsNum, FastUpdateArguments fastUpdateArgs, DistributedPlanInfo distributed) { this.colNames = colNames; this.colTypes = colTypes; this.rowsNum = rowsNum; @@ -93,46 +87,84 @@ private UpdatePlan(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] co this.selectQry = selectQry; this.isLocSubqry = isLocSubqry; this.fastUpdateArgs = fastUpdateArgs; + this.distributed = distributed; } /** */ public static UpdatePlan forMerge(GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry, - int rowsNum) { + int rowsNum, DistributedPlanInfo distributed) { assert !F.isEmpty(colNames); return new UpdatePlan(UpdateMode.MERGE, tbl, colNames, colTypes, keySupplier, valSupplier, keyColIdx, valColIdx, - selectQry, isLocSubqry, rowsNum, null); + selectQry, isLocSubqry, rowsNum, null, distributed); } /** */ public static UpdatePlan forInsert(GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry, - int rowsNum) { + int rowsNum, DistributedPlanInfo distributed) { assert !F.isEmpty(colNames); return new UpdatePlan(UpdateMode.INSERT, tbl, colNames, colTypes, keySupplier, valSupplier, keyColIdx, - valColIdx, selectQry, isLocSubqry, rowsNum, null); + valColIdx, selectQry, isLocSubqry, rowsNum, null, distributed); } /** */ public static UpdatePlan forUpdate(GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier valSupplier, - int valColIdx, String selectQry) { + int valColIdx, String selectQry, DistributedPlanInfo distributed) { assert !F.isEmpty(colNames); return new UpdatePlan(UpdateMode.UPDATE, tbl, colNames, colTypes, null, valSupplier, -1, valColIdx, selectQry, - false, 0, null); + false, 0, null, distributed); } /** */ - public static UpdatePlan forDelete(GridH2Table tbl, String selectQry) { - return new UpdatePlan(UpdateMode.DELETE, tbl, null, null, null, null, -1, -1, selectQry, false, 0, null); + public static UpdatePlan forDelete(GridH2Table tbl, String selectQry, DistributedPlanInfo distributed) { + return new UpdatePlan(UpdateMode.DELETE, tbl, null, null, null, null, -1, -1, selectQry, false, 0, null, + distributed); } /** */ public static UpdatePlan forFastUpdate(UpdateMode mode, GridH2Table tbl, FastUpdateArguments fastUpdateArgs) { assert mode == UpdateMode.UPDATE || mode == UpdateMode.DELETE; - return new UpdatePlan(mode, tbl, null, null, null, null, -1, -1, null, false, 0, fastUpdateArgs); + return new UpdatePlan(mode, tbl, null, null, null, null, -1, -1, null, false, 0, fastUpdateArgs, null); + } + + /** + * Additional information about distributed update plan. + */ + public final static class DistributedPlanInfo { + /** */ + private final boolean replicatedOnly; + + /** */ + private final List cacheIds; + + /** + * Constructor. + * + * @param replicatedOnly Whether all caches are replicated. + * @param cacheIds List of cache identifiers. + */ + DistributedPlanInfo(boolean replicatedOnly, List cacheIds) { + this.replicatedOnly = replicatedOnly; + this.cacheIds = cacheIds; + } + + /** + * @return {@code true} in case all caches are replicated. + */ + public boolean isReplicatedOnly() { + return replicatedOnly; + } + + /** + * @return cache identifiers. + */ + public List getCacheIds() { + return cacheIds; + } } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java index 804f7d8f61d7d..2bbbd9d2be024 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java @@ -18,6 +18,9 @@ package org.apache.ignite.internal.processors.query.h2.dml; import java.lang.reflect.Constructor; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -25,12 +28,15 @@ import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.binary.BinaryObjectBuilder; import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery; import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; +import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.query.GridQueryProperty; import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor; import org.apache.ignite.internal.processors.query.IgniteSQLException; import org.apache.ignite.internal.processors.query.QueryUtils; import org.apache.ignite.internal.processors.query.h2.DmlStatementsProcessor; +import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table; import org.apache.ignite.internal.processors.query.h2.sql.DmlAstUtils; @@ -41,12 +47,15 @@ import org.apache.ignite.internal.processors.query.h2.sql.GridSqlMerge; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuery; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser; +import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlSelect; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlStatement; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlUnion; import org.apache.ignite.internal.processors.query.h2.sql.GridSqlUpdate; import org.apache.ignite.internal.util.GridUnsafe; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.h2.command.Prepared; @@ -71,29 +80,39 @@ private UpdatePlanBuilder() { * if available. * * @param prepared H2's {@link Prepared}. + * @param loc Local query flag. + * @param idx Indexing. + * @param conn Connection. + * @param fieldsQuery Original query. * @return Update plan. */ - public static UpdatePlan planForStatement(Prepared prepared, - @Nullable Integer errKeysPos) throws IgniteCheckedException { + public static UpdatePlan planForStatement(Prepared prepared, boolean loc, IgniteH2Indexing idx, + @Nullable Connection conn, @Nullable SqlFieldsQueryEx fieldsQuery, @Nullable Integer errKeysPos) + throws IgniteCheckedException { assert !prepared.isQuery(); GridSqlStatement stmt = new GridSqlQueryParser(false).parse(prepared); if (stmt instanceof GridSqlMerge || stmt instanceof GridSqlInsert) - return planForInsert(stmt); + return planForInsert(stmt, loc, idx, conn, fieldsQuery); else - return planForUpdate(stmt, errKeysPos); + return planForUpdate(stmt, loc, idx, conn, fieldsQuery, errKeysPos); } /** * Prepare update plan for INSERT or MERGE. * * @param stmt INSERT or MERGE statement. + * @param loc Local query flag. + * @param idx Indexing. + * @param conn Connection. + * @param fieldsQuery Original query. * @return Update plan. * @throws IgniteCheckedException if failed. */ @SuppressWarnings("ConstantConditions") - private static UpdatePlan planForInsert(GridSqlStatement stmt) throws IgniteCheckedException { + private static UpdatePlan planForInsert(GridSqlStatement stmt, boolean loc, IgniteH2Indexing idx, + @Nullable Connection conn, @Nullable SqlFieldsQueryEx fieldsQuery) throws IgniteCheckedException { GridSqlQuery sel; GridSqlElement target; @@ -191,23 +210,33 @@ else if (stmt instanceof GridSqlMerge) { KeyValueSupplier keySupplier = createSupplier(cctx, desc.type(), keyColIdx, hasKeyProps, true, false); KeyValueSupplier valSupplier = createSupplier(cctx, desc.type(), valColIdx, hasValProps, false, false); + String selectSql = sel.getSQL(); + + UpdatePlan.DistributedPlanInfo distributed = (rowsNum == 0 && !F.isEmpty(selectSql)) ? + checkPlanCanBeDistributed(idx, conn, fieldsQuery, loc, selectSql, tbl.dataTable().cacheName()) : null; + if (stmt instanceof GridSqlMerge) return UpdatePlan.forMerge(tbl.dataTable(), colNames, colTypes, keySupplier, valSupplier, keyColIdx, - valColIdx, sel.getSQL(), !isTwoStepSubqry, rowsNum); + valColIdx, selectSql, !isTwoStepSubqry, rowsNum, distributed); else return UpdatePlan.forInsert(tbl.dataTable(), colNames, colTypes, keySupplier, valSupplier, keyColIdx, - valColIdx, sel.getSQL(), !isTwoStepSubqry, rowsNum); + valColIdx, selectSql, !isTwoStepSubqry, rowsNum, distributed); } /** * Prepare update plan for UPDATE or DELETE. * * @param stmt UPDATE or DELETE statement. + * @param loc Local query flag. + * @param idx Indexing. + * @param conn Connection. + * @param fieldsQuery Original query. * @param errKeysPos index to inject param for re-run keys at. Null if it's not a re-run plan. * @return Update plan. * @throws IgniteCheckedException if failed. */ - private static UpdatePlan planForUpdate(GridSqlStatement stmt, @Nullable Integer errKeysPos) + private static UpdatePlan planForUpdate(GridSqlStatement stmt, boolean loc, IgniteH2Indexing idx, + @Nullable Connection conn, @Nullable SqlFieldsQueryEx fieldsQuery, @Nullable Integer errKeysPos) throws IgniteCheckedException { GridSqlElement target; @@ -286,12 +315,23 @@ else if (stmt instanceof GridSqlDelete) { sel = DmlAstUtils.selectForUpdate((GridSqlUpdate) stmt, errKeysPos); - return UpdatePlan.forUpdate(gridTbl, colNames, colTypes, newValSupplier, valColIdx, sel.getSQL()); + String selectSql = sel.getSQL(); + + UpdatePlan.DistributedPlanInfo distributed = F.isEmpty(selectSql) ? null : + checkPlanCanBeDistributed(idx, conn, fieldsQuery, loc, selectSql, tbl.dataTable().cacheName()); + + return UpdatePlan.forUpdate(gridTbl, colNames, colTypes, newValSupplier, valColIdx, selectSql, + distributed); } else { sel = DmlAstUtils.selectForDelete((GridSqlDelete) stmt, errKeysPos); - return UpdatePlan.forDelete(gridTbl, sel.getSQL()); + String selectSql = sel.getSQL(); + + UpdatePlan.DistributedPlanInfo distributed = F.isEmpty(selectSql) ? null : + checkPlanCanBeDistributed(idx, conn, fieldsQuery, loc, selectSql, tbl.dataTable().cacheName()); + + return UpdatePlan.forDelete(gridTbl, selectSql, distributed); } } } @@ -493,6 +533,51 @@ private static boolean updateAffectsKeyColumns(GridH2Table gridTbl, Set return false; } + /** + * Checks whether the given update plan can be distributed and returns additional info. + * + * @param idx Indexing. + * @param conn Connection. + * @param fieldsQry Initial update query. + * @param loc Local query flag. + * @param selectQry Derived select query. + * @param cacheName Cache name. + * @return distributed update plan info, or {@code null} if cannot be distributed. + * @throws IgniteCheckedException if failed. + */ + private static UpdatePlan.DistributedPlanInfo checkPlanCanBeDistributed(IgniteH2Indexing idx, + Connection conn, SqlFieldsQueryEx fieldsQry, boolean loc, String selectQry, String cacheName) + throws IgniteCheckedException { + + if (fieldsQry == null || !fieldsQry.isUpdateOnServer() || loc || fieldsQry.isLocal()) + return null; + + assert conn != null; + + try { + // Get a new prepared statement for derived select query. + try (PreparedStatement stmt = conn.prepareStatement(selectQry)) { + idx.bindParameters(stmt, F.asList(fieldsQry.getArgs())); + + GridCacheTwoStepQuery qry = GridSqlQuerySplitter.split(conn, + GridSqlQueryParser.prepared(stmt), + fieldsQry.getArgs(), + fieldsQry.isCollocated(), + fieldsQry.isDistributedJoins(), + fieldsQry.isEnforceJoinOrder(), idx); + + boolean distributed = qry.skipMergeTable() && qry.mapQueries().size() == 1 && + !qry.mapQueries().get(0).hasSubQueries(); + + return distributed ? new UpdatePlan.DistributedPlanInfo(qry.isReplicatedOnly(), + idx.collectCacheIds(CU.cacheId(cacheName), qry)): null; + } + } + catch (SQLException e) { + throw new IgniteCheckedException(e); + } + } + /** * Simple supplier that just takes specified element of a given row. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 77b928f062e27..36504b03906fa 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -38,7 +38,6 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cache.query.QueryCancelledException; -import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.events.CacheQueryExecutedEvent; import org.apache.ignite.events.DiscoveryEvent; @@ -57,6 +56,7 @@ import org.apache.ignite.internal.processors.cache.query.CacheQueryType; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable; import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; +import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.query.GridQueryCancel; import org.apache.ignite.internal.processors.query.h2.H2Utils; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; @@ -776,7 +776,7 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th GridQueryCancel cancel = nodeResults.putUpdate(reqId); - SqlFieldsQuery fldsQry = new SqlFieldsQuery(req.query()); + SqlFieldsQueryEx fldsQry = new SqlFieldsQueryEx(req.query(), false); if (req.parameters() != null) fldsQry.setArgs(req.parameters()); From 22acb66962139be838188689cfd4aeca9dd4bcd2 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Wed, 11 Oct 2017 13:50:54 +0300 Subject: [PATCH 31/40] IGNITE-6024: Removed inner class from UpdatePlan --- .../query/h2/DmlStatementsProcessor.java | 7 +- .../processors/query/h2/dml/UpdatePlan.java | 71 ++++++------------- .../query/h2/dml/UpdatePlanBuilder.java | 27 ++++--- 3 files changed, 42 insertions(+), 63 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index b61c80a043da3..9b368d22545ea 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -374,7 +374,7 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo return doFastUpdate(plan, fieldsQry.getArgs()); } - if (plan.distributed != null) { + if (plan.distributed) { UpdateResult result = doDistributedUpdate(schemaName, fieldsQry, plan, cancel); // null is returned in case not all nodes support distributed DML. @@ -530,13 +530,10 @@ private static UpdateResult doFastUpdate(UpdatePlan plan, Object[] args) throws */ private UpdateResult doDistributedUpdate(String schemaName, SqlFieldsQuery fieldsQry, UpdatePlan plan, GridQueryCancel cancel) throws IgniteCheckedException { - assert plan.distributed != null; - if (cancel == null) cancel = new GridQueryCancel(); - return idx.runDistributedUpdate(schemaName, fieldsQry, plan.distributed.getCacheIds(), - plan.distributed.isReplicatedOnly(), cancel); + return idx.runDistributedUpdate(schemaName, fieldsQry, plan.cacheIds, plan.isReplicatedOnly, cancel); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index d09048e7c32eb..9b4a457234110 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -65,13 +65,20 @@ public final class UpdatePlan { /** Arguments for fast UPDATE or DELETE. */ public final FastUpdateArguments fastUpdateArgs; - /** Additional info for distributed update. */ - public final DistributedPlanInfo distributed; + /** Whether distributed update is possible. */ + public final boolean distributed; + + /** Whether update only uses replicated caches. */ + public final boolean isReplicatedOnly; + + /** Cache identifiers. */ + public List cacheIds; /** */ private UpdatePlan(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry, - int rowsNum, FastUpdateArguments fastUpdateArgs, DistributedPlanInfo distributed) { + int rowsNum, FastUpdateArguments fastUpdateArgs, boolean distributed, boolean isReplicatedOnly, + List cacheIds) { this.colNames = colNames; this.colTypes = colTypes; this.rowsNum = rowsNum; @@ -88,83 +95,51 @@ private UpdatePlan(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] co this.isLocSubqry = isLocSubqry; this.fastUpdateArgs = fastUpdateArgs; this.distributed = distributed; + this.isReplicatedOnly = isReplicatedOnly; + this.cacheIds = cacheIds; } /** */ public static UpdatePlan forMerge(GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry, - int rowsNum, DistributedPlanInfo distributed) { + int rowsNum, boolean distributed, boolean isReplicatedOnly, List cacheIds) { assert !F.isEmpty(colNames); return new UpdatePlan(UpdateMode.MERGE, tbl, colNames, colTypes, keySupplier, valSupplier, keyColIdx, valColIdx, - selectQry, isLocSubqry, rowsNum, null, distributed); + selectQry, isLocSubqry, rowsNum, null, distributed, isReplicatedOnly, cacheIds); } /** */ public static UpdatePlan forInsert(GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry, - int rowsNum, DistributedPlanInfo distributed) { + int rowsNum, boolean distributed, boolean isReplicatedOnly, List cacheIds) { assert !F.isEmpty(colNames); return new UpdatePlan(UpdateMode.INSERT, tbl, colNames, colTypes, keySupplier, valSupplier, keyColIdx, - valColIdx, selectQry, isLocSubqry, rowsNum, null, distributed); + valColIdx, selectQry, isLocSubqry, rowsNum, null, distributed, isReplicatedOnly, cacheIds); } /** */ public static UpdatePlan forUpdate(GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier valSupplier, - int valColIdx, String selectQry, DistributedPlanInfo distributed) { + int valColIdx, String selectQry, boolean distributed, boolean isReplicatedOnly, List cacheIds) { assert !F.isEmpty(colNames); return new UpdatePlan(UpdateMode.UPDATE, tbl, colNames, colTypes, null, valSupplier, -1, valColIdx, selectQry, - false, 0, null, distributed); + false, 0, null, distributed, isReplicatedOnly, cacheIds); } /** */ - public static UpdatePlan forDelete(GridH2Table tbl, String selectQry, DistributedPlanInfo distributed) { + public static UpdatePlan forDelete(GridH2Table tbl, String selectQry, boolean distributed, boolean isReplicatedOnly, + List cacheIds) { return new UpdatePlan(UpdateMode.DELETE, tbl, null, null, null, null, -1, -1, selectQry, false, 0, null, - distributed); + distributed, isReplicatedOnly, cacheIds); } /** */ public static UpdatePlan forFastUpdate(UpdateMode mode, GridH2Table tbl, FastUpdateArguments fastUpdateArgs) { assert mode == UpdateMode.UPDATE || mode == UpdateMode.DELETE; - return new UpdatePlan(mode, tbl, null, null, null, null, -1, -1, null, false, 0, fastUpdateArgs, null); - } - - /** - * Additional information about distributed update plan. - */ - public final static class DistributedPlanInfo { - /** */ - private final boolean replicatedOnly; - - /** */ - private final List cacheIds; - - /** - * Constructor. - * - * @param replicatedOnly Whether all caches are replicated. - * @param cacheIds List of cache identifiers. - */ - DistributedPlanInfo(boolean replicatedOnly, List cacheIds) { - this.replicatedOnly = replicatedOnly; - this.cacheIds = cacheIds; - } - - /** - * @return {@code true} in case all caches are replicated. - */ - public boolean isReplicatedOnly() { - return replicatedOnly; - } - - /** - * @return cache identifiers. - */ - public List getCacheIds() { - return cacheIds; - } + return new UpdatePlan(mode, tbl, null, null, null, null, -1, -1, null, false, 0, fastUpdateArgs, false, + false, null); } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java index 2bbbd9d2be024..1c9926a0d0560 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java @@ -58,6 +58,7 @@ import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiTuple; import org.h2.command.Prepared; import org.h2.table.Column; import org.jetbrains.annotations.Nullable; @@ -212,15 +213,18 @@ else if (stmt instanceof GridSqlMerge) { String selectSql = sel.getSQL(); - UpdatePlan.DistributedPlanInfo distributed = (rowsNum == 0 && !F.isEmpty(selectSql)) ? + IgniteBiTuple> distributed = (rowsNum == 0 && !F.isEmpty(selectSql)) ? checkPlanCanBeDistributed(idx, conn, fieldsQuery, loc, selectSql, tbl.dataTable().cacheName()) : null; + boolean isReplicatedOnly = distributed != null ? distributed.get1() : false; + List cacheIds = distributed != null ? distributed.get2() : null; + if (stmt instanceof GridSqlMerge) return UpdatePlan.forMerge(tbl.dataTable(), colNames, colTypes, keySupplier, valSupplier, keyColIdx, - valColIdx, selectSql, !isTwoStepSubqry, rowsNum, distributed); + valColIdx, selectSql, !isTwoStepSubqry, rowsNum, distributed != null, isReplicatedOnly, cacheIds); else return UpdatePlan.forInsert(tbl.dataTable(), colNames, colTypes, keySupplier, valSupplier, keyColIdx, - valColIdx, selectSql, !isTwoStepSubqry, rowsNum, distributed); + valColIdx, selectSql, !isTwoStepSubqry, rowsNum, distributed != null, isReplicatedOnly, cacheIds); } /** @@ -317,21 +321,24 @@ else if (stmt instanceof GridSqlDelete) { String selectSql = sel.getSQL(); - UpdatePlan.DistributedPlanInfo distributed = F.isEmpty(selectSql) ? null : + IgniteBiTuple> distributed = F.isEmpty(selectSql) ? null : checkPlanCanBeDistributed(idx, conn, fieldsQuery, loc, selectSql, tbl.dataTable().cacheName()); return UpdatePlan.forUpdate(gridTbl, colNames, colTypes, newValSupplier, valColIdx, selectSql, - distributed); + distributed != null, distributed != null ? distributed.get1() : false, + distributed != null ? distributed.get2() : null); } else { sel = DmlAstUtils.selectForDelete((GridSqlDelete) stmt, errKeysPos); String selectSql = sel.getSQL(); - UpdatePlan.DistributedPlanInfo distributed = F.isEmpty(selectSql) ? null : + IgniteBiTuple> distributed = F.isEmpty(selectSql) ? null : checkPlanCanBeDistributed(idx, conn, fieldsQuery, loc, selectSql, tbl.dataTable().cacheName()); - return UpdatePlan.forDelete(gridTbl, selectSql, distributed); + return UpdatePlan.forDelete(gridTbl, selectSql, distributed != null, + distributed != null ? distributed.get1() : false, + distributed != null ? distributed.get2() : null); } } } @@ -542,10 +549,10 @@ private static boolean updateAffectsKeyColumns(GridH2Table gridTbl, Set * @param loc Local query flag. * @param selectQry Derived select query. * @param cacheName Cache name. - * @return distributed update plan info, or {@code null} if cannot be distributed. + * @return Tuple containing additional info for distributed update or {@code null} if cannot be distributed. * @throws IgniteCheckedException if failed. */ - private static UpdatePlan.DistributedPlanInfo checkPlanCanBeDistributed(IgniteH2Indexing idx, + private static IgniteBiTuple> checkPlanCanBeDistributed(IgniteH2Indexing idx, Connection conn, SqlFieldsQueryEx fieldsQry, boolean loc, String selectQry, String cacheName) throws IgniteCheckedException { @@ -569,7 +576,7 @@ private static UpdatePlan.DistributedPlanInfo checkPlanCanBeDistributed(IgniteH2 boolean distributed = qry.skipMergeTable() && qry.mapQueries().size() == 1 && !qry.mapQueries().get(0).hasSubQueries(); - return distributed ? new UpdatePlan.DistributedPlanInfo(qry.isReplicatedOnly(), + return distributed ? new IgniteBiTuple<>(qry.isReplicatedOnly(), idx.collectCacheIds(CU.cacheId(cacheName), qry)): null; } } From 94ce00d90bb7536d28842df1c40617d778f36dfb Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Wed, 11 Oct 2017 14:05:27 +0300 Subject: [PATCH 32/40] IGNITE-6024: applied review comment --- .../ignite/internal/processors/query/h2/dml/UpdatePlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 9b4a457234110..7c699cadec76b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -72,7 +72,7 @@ public final class UpdatePlan { public final boolean isReplicatedOnly; /** Cache identifiers. */ - public List cacheIds; + public final List cacheIds; /** */ private UpdatePlan(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, From 3f9bbb4bb6a97c953026ce513adb45122a48ddd9 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Wed, 11 Oct 2017 14:20:10 +0300 Subject: [PATCH 33/40] IGNITE-6024: minor --- .../ignite/internal/processors/query/h2/dml/UpdatePlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 7c699cadec76b..5576aa792dc96 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -71,7 +71,7 @@ public final class UpdatePlan { /** Whether update only uses replicated caches. */ public final boolean isReplicatedOnly; - /** Cache identifiers. */ + /** Identifiers of caches involved in update (Used for cluster nodes mapping in case of distributed update). */ public final List cacheIds; /** */ From 7218fd044dbaacf9b7e6d4f6a25980f87fa443d9 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Wed, 11 Oct 2017 17:28:04 +0300 Subject: [PATCH 34/40] Revert "IGNITE-6024: minor" This reverts commit 3f9bbb4bb6a97c953026ce513adb45122a48ddd9. --- .../ignite/internal/processors/query/h2/dml/UpdatePlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 5576aa792dc96..7c699cadec76b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -71,7 +71,7 @@ public final class UpdatePlan { /** Whether update only uses replicated caches. */ public final boolean isReplicatedOnly; - /** Identifiers of caches involved in update (Used for cluster nodes mapping in case of distributed update). */ + /** Cache identifiers. */ public final List cacheIds; /** */ From 4c3d78168ba8e8f5d6bcb83b50032604c681d6f2 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Wed, 11 Oct 2017 17:28:14 +0300 Subject: [PATCH 35/40] Revert "IGNITE-6024: applied review comment" This reverts commit 94ce00d90bb7536d28842df1c40617d778f36dfb. --- .../ignite/internal/processors/query/h2/dml/UpdatePlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 7c699cadec76b..9b4a457234110 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -72,7 +72,7 @@ public final class UpdatePlan { public final boolean isReplicatedOnly; /** Cache identifiers. */ - public final List cacheIds; + public List cacheIds; /** */ private UpdatePlan(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, From b27e30edad27986fad8c008dc5b1f409b049991d Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Wed, 11 Oct 2017 17:28:30 +0300 Subject: [PATCH 36/40] Revert "IGNITE-6024: Removed inner class from UpdatePlan" This reverts commit 22acb66962139be838188689cfd4aeca9dd4bcd2. --- .../query/h2/DmlStatementsProcessor.java | 7 +- .../processors/query/h2/dml/UpdatePlan.java | 71 +++++++++++++------ .../query/h2/dml/UpdatePlanBuilder.java | 27 +++---- 3 files changed, 63 insertions(+), 42 deletions(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index 9b368d22545ea..b61c80a043da3 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -374,7 +374,7 @@ private UpdateResult executeUpdateStatement(String schemaName, final GridCacheCo return doFastUpdate(plan, fieldsQry.getArgs()); } - if (plan.distributed) { + if (plan.distributed != null) { UpdateResult result = doDistributedUpdate(schemaName, fieldsQry, plan, cancel); // null is returned in case not all nodes support distributed DML. @@ -530,10 +530,13 @@ private static UpdateResult doFastUpdate(UpdatePlan plan, Object[] args) throws */ private UpdateResult doDistributedUpdate(String schemaName, SqlFieldsQuery fieldsQry, UpdatePlan plan, GridQueryCancel cancel) throws IgniteCheckedException { + assert plan.distributed != null; + if (cancel == null) cancel = new GridQueryCancel(); - return idx.runDistributedUpdate(schemaName, fieldsQry, plan.cacheIds, plan.isReplicatedOnly, cancel); + return idx.runDistributedUpdate(schemaName, fieldsQry, plan.distributed.getCacheIds(), + plan.distributed.isReplicatedOnly(), cancel); } /** diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index 9b4a457234110..d09048e7c32eb 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -65,20 +65,13 @@ public final class UpdatePlan { /** Arguments for fast UPDATE or DELETE. */ public final FastUpdateArguments fastUpdateArgs; - /** Whether distributed update is possible. */ - public final boolean distributed; - - /** Whether update only uses replicated caches. */ - public final boolean isReplicatedOnly; - - /** Cache identifiers. */ - public List cacheIds; + /** Additional info for distributed update. */ + public final DistributedPlanInfo distributed; /** */ private UpdatePlan(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry, - int rowsNum, FastUpdateArguments fastUpdateArgs, boolean distributed, boolean isReplicatedOnly, - List cacheIds) { + int rowsNum, FastUpdateArguments fastUpdateArgs, DistributedPlanInfo distributed) { this.colNames = colNames; this.colTypes = colTypes; this.rowsNum = rowsNum; @@ -95,51 +88,83 @@ private UpdatePlan(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] co this.isLocSubqry = isLocSubqry; this.fastUpdateArgs = fastUpdateArgs; this.distributed = distributed; - this.isReplicatedOnly = isReplicatedOnly; - this.cacheIds = cacheIds; } /** */ public static UpdatePlan forMerge(GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry, - int rowsNum, boolean distributed, boolean isReplicatedOnly, List cacheIds) { + int rowsNum, DistributedPlanInfo distributed) { assert !F.isEmpty(colNames); return new UpdatePlan(UpdateMode.MERGE, tbl, colNames, colTypes, keySupplier, valSupplier, keyColIdx, valColIdx, - selectQry, isLocSubqry, rowsNum, null, distributed, isReplicatedOnly, cacheIds); + selectQry, isLocSubqry, rowsNum, null, distributed); } /** */ public static UpdatePlan forInsert(GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry, - int rowsNum, boolean distributed, boolean isReplicatedOnly, List cacheIds) { + int rowsNum, DistributedPlanInfo distributed) { assert !F.isEmpty(colNames); return new UpdatePlan(UpdateMode.INSERT, tbl, colNames, colTypes, keySupplier, valSupplier, keyColIdx, - valColIdx, selectQry, isLocSubqry, rowsNum, null, distributed, isReplicatedOnly, cacheIds); + valColIdx, selectQry, isLocSubqry, rowsNum, null, distributed); } /** */ public static UpdatePlan forUpdate(GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier valSupplier, - int valColIdx, String selectQry, boolean distributed, boolean isReplicatedOnly, List cacheIds) { + int valColIdx, String selectQry, DistributedPlanInfo distributed) { assert !F.isEmpty(colNames); return new UpdatePlan(UpdateMode.UPDATE, tbl, colNames, colTypes, null, valSupplier, -1, valColIdx, selectQry, - false, 0, null, distributed, isReplicatedOnly, cacheIds); + false, 0, null, distributed); } /** */ - public static UpdatePlan forDelete(GridH2Table tbl, String selectQry, boolean distributed, boolean isReplicatedOnly, - List cacheIds) { + public static UpdatePlan forDelete(GridH2Table tbl, String selectQry, DistributedPlanInfo distributed) { return new UpdatePlan(UpdateMode.DELETE, tbl, null, null, null, null, -1, -1, selectQry, false, 0, null, - distributed, isReplicatedOnly, cacheIds); + distributed); } /** */ public static UpdatePlan forFastUpdate(UpdateMode mode, GridH2Table tbl, FastUpdateArguments fastUpdateArgs) { assert mode == UpdateMode.UPDATE || mode == UpdateMode.DELETE; - return new UpdatePlan(mode, tbl, null, null, null, null, -1, -1, null, false, 0, fastUpdateArgs, false, - false, null); + return new UpdatePlan(mode, tbl, null, null, null, null, -1, -1, null, false, 0, fastUpdateArgs, null); + } + + /** + * Additional information about distributed update plan. + */ + public final static class DistributedPlanInfo { + /** */ + private final boolean replicatedOnly; + + /** */ + private final List cacheIds; + + /** + * Constructor. + * + * @param replicatedOnly Whether all caches are replicated. + * @param cacheIds List of cache identifiers. + */ + DistributedPlanInfo(boolean replicatedOnly, List cacheIds) { + this.replicatedOnly = replicatedOnly; + this.cacheIds = cacheIds; + } + + /** + * @return {@code true} in case all caches are replicated. + */ + public boolean isReplicatedOnly() { + return replicatedOnly; + } + + /** + * @return cache identifiers. + */ + public List getCacheIds() { + return cacheIds; + } } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java index 1c9926a0d0560..2bbbd9d2be024 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java @@ -58,7 +58,6 @@ import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.lang.IgniteBiTuple; import org.h2.command.Prepared; import org.h2.table.Column; import org.jetbrains.annotations.Nullable; @@ -213,18 +212,15 @@ else if (stmt instanceof GridSqlMerge) { String selectSql = sel.getSQL(); - IgniteBiTuple> distributed = (rowsNum == 0 && !F.isEmpty(selectSql)) ? + UpdatePlan.DistributedPlanInfo distributed = (rowsNum == 0 && !F.isEmpty(selectSql)) ? checkPlanCanBeDistributed(idx, conn, fieldsQuery, loc, selectSql, tbl.dataTable().cacheName()) : null; - boolean isReplicatedOnly = distributed != null ? distributed.get1() : false; - List cacheIds = distributed != null ? distributed.get2() : null; - if (stmt instanceof GridSqlMerge) return UpdatePlan.forMerge(tbl.dataTable(), colNames, colTypes, keySupplier, valSupplier, keyColIdx, - valColIdx, selectSql, !isTwoStepSubqry, rowsNum, distributed != null, isReplicatedOnly, cacheIds); + valColIdx, selectSql, !isTwoStepSubqry, rowsNum, distributed); else return UpdatePlan.forInsert(tbl.dataTable(), colNames, colTypes, keySupplier, valSupplier, keyColIdx, - valColIdx, selectSql, !isTwoStepSubqry, rowsNum, distributed != null, isReplicatedOnly, cacheIds); + valColIdx, selectSql, !isTwoStepSubqry, rowsNum, distributed); } /** @@ -321,24 +317,21 @@ else if (stmt instanceof GridSqlDelete) { String selectSql = sel.getSQL(); - IgniteBiTuple> distributed = F.isEmpty(selectSql) ? null : + UpdatePlan.DistributedPlanInfo distributed = F.isEmpty(selectSql) ? null : checkPlanCanBeDistributed(idx, conn, fieldsQuery, loc, selectSql, tbl.dataTable().cacheName()); return UpdatePlan.forUpdate(gridTbl, colNames, colTypes, newValSupplier, valColIdx, selectSql, - distributed != null, distributed != null ? distributed.get1() : false, - distributed != null ? distributed.get2() : null); + distributed); } else { sel = DmlAstUtils.selectForDelete((GridSqlDelete) stmt, errKeysPos); String selectSql = sel.getSQL(); - IgniteBiTuple> distributed = F.isEmpty(selectSql) ? null : + UpdatePlan.DistributedPlanInfo distributed = F.isEmpty(selectSql) ? null : checkPlanCanBeDistributed(idx, conn, fieldsQuery, loc, selectSql, tbl.dataTable().cacheName()); - return UpdatePlan.forDelete(gridTbl, selectSql, distributed != null, - distributed != null ? distributed.get1() : false, - distributed != null ? distributed.get2() : null); + return UpdatePlan.forDelete(gridTbl, selectSql, distributed); } } } @@ -549,10 +542,10 @@ private static boolean updateAffectsKeyColumns(GridH2Table gridTbl, Set * @param loc Local query flag. * @param selectQry Derived select query. * @param cacheName Cache name. - * @return Tuple containing additional info for distributed update or {@code null} if cannot be distributed. + * @return distributed update plan info, or {@code null} if cannot be distributed. * @throws IgniteCheckedException if failed. */ - private static IgniteBiTuple> checkPlanCanBeDistributed(IgniteH2Indexing idx, + private static UpdatePlan.DistributedPlanInfo checkPlanCanBeDistributed(IgniteH2Indexing idx, Connection conn, SqlFieldsQueryEx fieldsQry, boolean loc, String selectQry, String cacheName) throws IgniteCheckedException { @@ -576,7 +569,7 @@ private static IgniteBiTuple> checkPlanCanBeDistributed(I boolean distributed = qry.skipMergeTable() && qry.mapQueries().size() == 1 && !qry.mapQueries().get(0).hasSubQueries(); - return distributed ? new IgniteBiTuple<>(qry.isReplicatedOnly(), + return distributed ? new UpdatePlan.DistributedPlanInfo(qry.isReplicatedOnly(), idx.collectCacheIds(CU.cacheId(cacheName), qry)): null; } } From 887b6616cf3552a486177372990c946a1fa4e5f4 Mon Sep 17 00:00:00 2001 From: skalashnikov Date: Wed, 11 Oct 2017 18:17:00 +0300 Subject: [PATCH 37/40] IGNITE-6024: applied review comments. --- .../ignite/cache/query/SqlFieldsQuery.java | 7 ++++++ .../cache/query/SqlFieldsQueryEx.java | 19 ++++++++-------- .../query/h2/DmlStatementsProcessor.java | 13 +++++------ .../processors/query/h2/H2DmlPlanKey.java | 7 +++--- .../processors/query/h2/IgniteH2Indexing.java | 6 ++--- .../processors/query/h2/dml/UpdatePlan.java | 6 ++--- .../query/h2/dml/UpdatePlanBuilder.java | 22 ++++++++++++++----- .../h2/twostep/GridMapQueryExecutor.java | 4 ++-- 8 files changed, 51 insertions(+), 33 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/cache/query/SqlFieldsQuery.java b/modules/core/src/main/java/org/apache/ignite/cache/query/SqlFieldsQuery.java index 2d128d158b9fb..4e12b8ca03ecc 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/query/SqlFieldsQuery.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/query/SqlFieldsQuery.java @@ -369,6 +369,13 @@ public SqlFieldsQuery setSchema(@Nullable String schema) { return this; } + /** + * @return Copy of this query. + */ + public SqlFieldsQuery copy() { + return new SqlFieldsQuery(this); + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(SqlFieldsQuery.class, this); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java index 9c77063948303..343078b600a48 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java @@ -45,17 +45,11 @@ public SqlFieldsQueryEx(String sql, Boolean isQry) { /** * @param qry SQL query. */ - public SqlFieldsQueryEx(SqlFieldsQuery qry) { + public SqlFieldsQueryEx(SqlFieldsQueryEx qry) { super(qry); - if (qry instanceof SqlFieldsQueryEx) { - SqlFieldsQueryEx other = (SqlFieldsQueryEx)qry; - - this.isQry = other.isQry; - this.updateOnServer = other.updateOnServer; - } - else - this.isQry = null; + this.isQry = qry.isQry; + this.updateOnServer = qry.updateOnServer; } /** @@ -125,7 +119,7 @@ public Boolean isQuery() { * Sets server side update flag. *

* By default, when processing DML command, Ignite first fetches all affected intermediate rows for analysis to the - * node which initiated the query and only then forms batches of updated values to be send to remote nodes. + * node which initiated the query and only then forms batches of updated values to be sent to remote nodes. * For simple DML commands (that however affect great deal of rows) such approach may be an overkill in terms of * network delays and memory usage on initiating node. Use this flag as hint for Ignite to do all intermediate rows * analysis and updates in place on corresponding remote data nodes. @@ -156,4 +150,9 @@ public SqlFieldsQuery setUpdateOnServer(boolean updateOnServer) { public boolean isUpdateOnServer() { return updateOnServer; } + + /** {@inheritDoc} */ + @Override public SqlFieldsQuery copy() { + return new SqlFieldsQueryEx(this); + } } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java index b61c80a043da3..9e55442a422d1 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java @@ -50,7 +50,6 @@ import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.GridKernalContext; -import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheOperationContext; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; @@ -162,7 +161,7 @@ public void onCacheStop(String cacheName) { * @throws IgniteCheckedException if failed. */ private UpdateResult updateSqlFields(String schemaName, Connection conn, Prepared prepared, - SqlFieldsQueryEx fieldsQry, boolean loc, IndexingQueryFilter filters, GridQueryCancel cancel) + SqlFieldsQuery fieldsQry, boolean loc, IndexingQueryFilter filters, GridQueryCancel cancel) throws IgniteCheckedException { Object[] errKeys = null; @@ -226,7 +225,7 @@ else if (items == 0L) */ @SuppressWarnings("unchecked") QueryCursorImpl> updateSqlFieldsDistributed(String schemaName, Connection c, Prepared p, - SqlFieldsQueryEx fieldsQry, GridQueryCancel cancel) throws IgniteCheckedException { + SqlFieldsQuery fieldsQry, GridQueryCancel cancel) throws IgniteCheckedException { UpdateResult res = updateSqlFields(schemaName, c, p, fieldsQry, false, null, cancel); checkUpdateResult(res); @@ -253,7 +252,7 @@ QueryCursorImpl> updateSqlFieldsDistributed(String schemaName, Connectio */ @SuppressWarnings("unchecked") GridQueryFieldsResult updateSqlFieldsLocal(String schemaName, Connection conn, PreparedStatement stmt, - SqlFieldsQueryEx fieldsQry, IndexingQueryFilter filters, GridQueryCancel cancel) + SqlFieldsQuery fieldsQry, IndexingQueryFilter filters, GridQueryCancel cancel) throws IgniteCheckedException { UpdateResult res = updateSqlFields(schemaName, conn, GridSqlQueryParser.prepared(stmt), fieldsQry, true, filters, cancel); @@ -360,7 +359,7 @@ long streamUpdateQuery(IgniteDataStreamer streamer, PreparedStatement stmt, Obje */ @SuppressWarnings({"ConstantConditions", "unchecked"}) private UpdateResult executeUpdateStatement(String schemaName, final GridCacheContext cctx, Connection c, - Prepared prepared, SqlFieldsQueryEx fieldsQry, boolean loc, IndexingQueryFilter filters, + Prepared prepared, SqlFieldsQuery fieldsQry, boolean loc, IndexingQueryFilter filters, GridQueryCancel cancel, Object[] failedKeys) throws IgniteCheckedException { int mainCacheId = CU.cacheId(cctx.name()); @@ -462,7 +461,7 @@ private UpdateResult processDmlSelectResult(GridCacheContext cctx, UpdatePlan pl * @return Update plan. */ @SuppressWarnings({"unchecked", "ConstantConditions"}) - private UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared p, SqlFieldsQueryEx fieldsQry, + private UpdatePlan getPlanForStatement(String schema, Connection conn, Prepared p, SqlFieldsQuery fieldsQry, boolean loc, @Nullable Integer errKeysPos) throws IgniteCheckedException { H2DmlPlanKey planKey = new H2DmlPlanKey(schema, p.getSQL(), loc, fieldsQry); @@ -1045,7 +1044,7 @@ private static PageProcessingResult processPage(GridCacheContext cctx, * @return Update result. * @throws IgniteCheckedException if failed. */ - UpdateResult mapDistributedUpdate(String schemaName, PreparedStatement stmt, SqlFieldsQueryEx fldsQry, + UpdateResult mapDistributedUpdate(String schemaName, PreparedStatement stmt, SqlFieldsQuery fldsQry, IndexingQueryFilter filter, GridQueryCancel cancel, boolean local) throws IgniteCheckedException { Connection c; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java index ea957869f795e..cbee2756bcd90 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java @@ -17,7 +17,8 @@ package org.apache.ignite.internal.processors.query.h2; -import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; +import org.apache.ignite.cache.query.SqlFieldsQuery; +import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlanBuilder; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.S; @@ -40,11 +41,11 @@ public class H2DmlPlanKey { * @param schemaName Schema name. * @param sql SQL. */ - public H2DmlPlanKey(String schemaName, String sql, boolean loc, SqlFieldsQueryEx fieldsQry) { + public H2DmlPlanKey(String schemaName, String sql, boolean loc, SqlFieldsQuery fieldsQry) { this.schemaName = schemaName; this.sql = sql; - if (!fieldsQry.isUpdateOnServer() || loc || fieldsQry.isLocal()) + if (loc || !UpdatePlanBuilder.isUpdateOnServerQuery(fieldsQry)) this.flags = 0; // flags only relevant for server side updates. else { this.flags = (byte)(1 + diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java index e05fb40695835..fddd2e8744b09 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java @@ -826,7 +826,7 @@ public GridQueryFieldsResult queryLocalSqlFields(final String schemaName, final Prepared p = GridSqlQueryParser.prepared(stmt); if (DmlStatementsProcessor.isDmlStatement(p)) { - SqlFieldsQueryEx fldsQry = new SqlFieldsQueryEx(qry, false); + SqlFieldsQuery fldsQry = new SqlFieldsQuery(qry); if (params != null) fldsQry.setArgs(params.toArray()); @@ -1451,7 +1451,7 @@ UpdateResult runDistributedUpdate( if (DmlStatementsProcessor.isDmlStatement(prepared)) { try { res.add(dmlProc.updateSqlFieldsDistributed(schemaName, c, prepared, - new SqlFieldsQueryEx(qry).setSql(sqlQry).setArgs(args), cancel)); + qry.copy().setSql(sqlQry).setArgs(args), cancel)); continue; } @@ -1580,7 +1580,7 @@ private FieldsQueryCursor> executeTwoStepsQuery(String schemaName, int p * @return Update result. * @throws IgniteCheckedException if failed. */ - public UpdateResult mapDistributedUpdate(String schemaName, SqlFieldsQueryEx fldsQry, IndexingQueryFilter filter, + public UpdateResult mapDistributedUpdate(String schemaName, SqlFieldsQuery fldsQry, IndexingQueryFilter filter, GridQueryCancel cancel, boolean local) throws IgniteCheckedException { Connection conn = connectionForSchema(schemaName); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java index d09048e7c32eb..a99d811cacbe5 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java @@ -136,10 +136,10 @@ public static UpdatePlan forFastUpdate(UpdateMode mode, GridH2Table tbl, FastUpd * Additional information about distributed update plan. */ public final static class DistributedPlanInfo { - /** */ + /** Whether update involves only replicated caches. */ private final boolean replicatedOnly; - /** */ + /** Identifiers of caches involved in update (used for cluster nodes mapping). */ private final List cacheIds; /** @@ -154,7 +154,7 @@ public final static class DistributedPlanInfo { } /** - * @return {@code true} in case all caches are replicated. + * @return {@code true} in case all involved caches are replicated. */ public boolean isReplicatedOnly() { return replicatedOnly; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java index 2bbbd9d2be024..ead4629ba4cf6 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java @@ -27,6 +27,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.binary.BinaryObjectBuilder; +import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery; import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode; @@ -87,7 +88,7 @@ private UpdatePlanBuilder() { * @return Update plan. */ public static UpdatePlan planForStatement(Prepared prepared, boolean loc, IgniteH2Indexing idx, - @Nullable Connection conn, @Nullable SqlFieldsQueryEx fieldsQuery, @Nullable Integer errKeysPos) + @Nullable Connection conn, @Nullable SqlFieldsQuery fieldsQuery, @Nullable Integer errKeysPos) throws IgniteCheckedException { assert !prepared.isQuery(); @@ -112,7 +113,7 @@ public static UpdatePlan planForStatement(Prepared prepared, boolean loc, Ignite */ @SuppressWarnings("ConstantConditions") private static UpdatePlan planForInsert(GridSqlStatement stmt, boolean loc, IgniteH2Indexing idx, - @Nullable Connection conn, @Nullable SqlFieldsQueryEx fieldsQuery) throws IgniteCheckedException { + @Nullable Connection conn, @Nullable SqlFieldsQuery fieldsQuery) throws IgniteCheckedException { GridSqlQuery sel; GridSqlElement target; @@ -236,7 +237,7 @@ else if (stmt instanceof GridSqlMerge) { * @throws IgniteCheckedException if failed. */ private static UpdatePlan planForUpdate(GridSqlStatement stmt, boolean loc, IgniteH2Indexing idx, - @Nullable Connection conn, @Nullable SqlFieldsQueryEx fieldsQuery, @Nullable Integer errKeysPos) + @Nullable Connection conn, @Nullable SqlFieldsQuery fieldsQuery, @Nullable Integer errKeysPos) throws IgniteCheckedException { GridSqlElement target; @@ -546,10 +547,10 @@ private static boolean updateAffectsKeyColumns(GridH2Table gridTbl, Set * @throws IgniteCheckedException if failed. */ private static UpdatePlan.DistributedPlanInfo checkPlanCanBeDistributed(IgniteH2Indexing idx, - Connection conn, SqlFieldsQueryEx fieldsQry, boolean loc, String selectQry, String cacheName) + Connection conn, SqlFieldsQuery fieldsQry, boolean loc, String selectQry, String cacheName) throws IgniteCheckedException { - if (fieldsQry == null || !fieldsQry.isUpdateOnServer() || loc || fieldsQry.isLocal()) + if (loc || !isUpdateOnServerQuery(fieldsQry)) return null; assert conn != null; @@ -578,6 +579,17 @@ private static UpdatePlan.DistributedPlanInfo checkPlanCanBeDistributed(IgniteH2 } } + /** + * Checks whether query flags are compatible with server side update. + * + * @param qry Query. + * @return {@code true} if update can be distributed. + */ + public static boolean isUpdateOnServerQuery(SqlFieldsQuery qry) { + return qry != null && !qry.isLocal() && + qry instanceof SqlFieldsQueryEx && ((SqlFieldsQueryEx)qry).isUpdateOnServer(); + } + /** * Simple supplier that just takes specified element of a given row. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 36504b03906fa..77b928f062e27 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -38,6 +38,7 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cache.query.QueryCancelledException; +import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.events.CacheQueryExecutedEvent; import org.apache.ignite.events.DiscoveryEvent; @@ -56,7 +57,6 @@ import org.apache.ignite.internal.processors.cache.query.CacheQueryType; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable; import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; -import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; import org.apache.ignite.internal.processors.query.GridQueryCancel; import org.apache.ignite.internal.processors.query.h2.H2Utils; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; @@ -776,7 +776,7 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th GridQueryCancel cancel = nodeResults.putUpdate(reqId); - SqlFieldsQueryEx fldsQry = new SqlFieldsQueryEx(req.query(), false); + SqlFieldsQuery fldsQry = new SqlFieldsQuery(req.query()); if (req.parameters() != null) fldsQry.setArgs(req.parameters()); From 1168bb99b386ca1cb3767277ea0c1a932077af38 Mon Sep 17 00:00:00 2001 From: devozerov Date: Thu, 12 Oct 2017 09:45:51 +0300 Subject: [PATCH 38/40] Cosmetics. --- .../internal/processors/cache/query/SqlFieldsQueryEx.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java index 343078b600a48..3a7c6a6dab052 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java @@ -45,7 +45,7 @@ public SqlFieldsQueryEx(String sql, Boolean isQry) { /** * @param qry SQL query. */ - public SqlFieldsQueryEx(SqlFieldsQueryEx qry) { + private SqlFieldsQueryEx(SqlFieldsQueryEx qry) { super(qry); this.isQry = qry.isQry; From 6f5134b02b074221c8d2f39638ae434f7d16f399 Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 13 Oct 2017 10:38:02 +0300 Subject: [PATCH 39/40] Reworked JDBC and ODBC drivers to new parameter name. --- .../jdbc2/JdbcConnectionSelfTest.java | 13 ++--- .../jdbc/suite/IgniteJdbcDriverTestSuite.java | 16 +++--- .../jdbc/thin/JdbcThinConnectionSelfTest.java | 20 +++---- ...lexDmlDdlSkipReducerOnUpdateSelfTest.java} | 4 +- ...StatementSkipReducerOnUpdateSelfTest.java} | 4 +- ...StatementSkipReducerOnUpdateSelfTest.java} | 4 +- ...StatementSkipReducerOnUpdateSelfTest.java} | 4 +- .../org/apache/ignite/IgniteJdbcDriver.java | 10 ++-- .../apache/ignite/IgniteJdbcThinDriver.java | 2 +- .../jdbc/thin/JdbcThinConnection.java | 6 +-- .../internal/jdbc/thin/JdbcThinTcpIo.java | 16 +++--- .../internal/jdbc/thin/JdbcThinUtils.java | 4 +- .../ignite/internal/jdbc2/JdbcConnection.java | 12 ++--- .../ignite/internal/jdbc2/JdbcQueryTask.java | 2 +- .../internal/jdbc2/JdbcQueryTaskV3.java | 20 +++---- .../ignite/internal/jdbc2/JdbcStatement.java | 2 +- .../cache/query/SqlFieldsQueryEx.java | 14 ++--- .../odbc/jdbc/JdbcConnectionContext.java | 6 +-- .../odbc/jdbc/JdbcRequestHandler.java | 15 +++--- .../odbc/odbc/OdbcConnectionContext.java | 6 +-- .../odbc/odbc/OdbcRequestHandler.java | 10 ++-- .../query/h2/dml/UpdatePlanBuilder.java | 2 +- .../IgniteSqlDistributedDmlFlagSelfTest.java | 4 +- .../IgniteSqlDistributedDmlSelfTest.java | 18 +++---- .../cpp/odbc-test/src/configuration_test.cpp | 24 ++++----- .../ignite/odbc/config/configuration.h | 16 +++--- .../cpp/odbc/include/ignite/odbc/message.h | 8 +-- .../odbc/system/ui/dsn_configuration_window.h | 4 +- .../system/ui/dsn_configuration_window.cpp | 23 ++++---- .../cpp/odbc/src/config/configuration.cpp | 52 +++++++++---------- modules/platforms/cpp/odbc/src/connection.cpp | 6 +-- modules/platforms/cpp/odbc/src/dsn_config.cpp | 5 +- modules/platforms/cpp/odbc/src/message.cpp | 6 +-- 33 files changed, 182 insertions(+), 176 deletions(-) rename modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/{JdbcThinComplexDmlDdlUpdateOnServerSelfTest.java => JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.java} (89%) rename modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/{JdbcThinInsertStatementUpdateOnServerSelfTest.java => JdbcThinInsertStatementSkipReducerOnUpdateSelfTest.java} (89%) rename modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/{JdbcThinMergeStatementUpdateOnServerSelfTest.java => JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.java} (89%) rename modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/{JdbcThinUpdateStatementUpdateOnServerSelfTest.java => JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest.java} (89%) diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java index 0940465e5f286..35d0fba0aa83b 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcConnectionSelfTest.java @@ -314,7 +314,7 @@ public void testSqlHints() throws Exception { assertFalse(((JdbcConnection)conn).isDistributedJoins()); assertFalse(((JdbcConnection)conn).isCollocatedQuery()); assertFalse(((JdbcConnection)conn).isLazy()); - assertFalse(((JdbcConnection)conn).updateOnServer()); + assertFalse(((JdbcConnection)conn).skipReducerOnUpdate()); } try (final Connection conn = DriverManager.getConnection(CFG_URL_PREFIX + "distributedJoins=true@" @@ -323,7 +323,7 @@ public void testSqlHints() throws Exception { assertTrue(((JdbcConnection)conn).isDistributedJoins()); assertFalse(((JdbcConnection)conn).isCollocatedQuery()); assertFalse(((JdbcConnection)conn).isLazy()); - assertFalse(((JdbcConnection)conn).updateOnServer()); + assertFalse(((JdbcConnection)conn).skipReducerOnUpdate()); } try (final Connection conn = DriverManager.getConnection(CFG_URL_PREFIX + "collocated=true@" @@ -332,7 +332,7 @@ public void testSqlHints() throws Exception { assertFalse(((JdbcConnection)conn).isDistributedJoins()); assertTrue(((JdbcConnection)conn).isCollocatedQuery()); assertFalse(((JdbcConnection)conn).isLazy()); - assertFalse(((JdbcConnection)conn).updateOnServer()); + assertFalse(((JdbcConnection)conn).skipReducerOnUpdate()); } try (final Connection conn = DriverManager.getConnection(CFG_URL_PREFIX + "lazy=true@" + configURL())) { @@ -340,14 +340,15 @@ public void testSqlHints() throws Exception { assertFalse(((JdbcConnection)conn).isDistributedJoins()); assertFalse(((JdbcConnection)conn).isCollocatedQuery()); assertTrue(((JdbcConnection)conn).isLazy()); - assertFalse(((JdbcConnection)conn).updateOnServer()); + assertFalse(((JdbcConnection)conn).skipReducerOnUpdate()); } - try (final Connection conn = DriverManager.getConnection(CFG_URL_PREFIX + "updateOnServer=true@" + configURL())) { + try (final Connection conn = DriverManager.getConnection(CFG_URL_PREFIX + "skipReducerOnUpdate=true@" + + configURL())) { assertFalse(((JdbcConnection)conn).isEnforceJoinOrder()); assertFalse(((JdbcConnection)conn).isDistributedJoins()); assertFalse(((JdbcConnection)conn).isCollocatedQuery()); assertFalse(((JdbcConnection)conn).isLazy()); - assertTrue(((JdbcConnection)conn).updateOnServer()); + assertTrue(((JdbcConnection)conn).skipReducerOnUpdate()); } } } diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java index f80fb615da1c8..c631af41870f4 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java @@ -58,10 +58,10 @@ import org.apache.ignite.jdbc.thin.JdbcThinSelectAfterAlterTable; import org.apache.ignite.jdbc.thin.JdbcThinStatementSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSelfTest; -import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinComplexDmlDdlUpdateOnServerSelfTest; -import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinInsertStatementUpdateOnServerSelfTest; -import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinMergeStatementUpdateOnServerSelfTest; -import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinUpdateStatementUpdateOnServerSelfTest; +import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest; +import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinInsertStatementSkipReducerOnUpdateSelfTest; +import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinMergeStatementSkipReducerOnUpdateSelfTest; +import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest; /** * JDBC driver test suite. @@ -157,10 +157,10 @@ public static TestSuite suite() throws Exception { suite.addTest(new TestSuite(JdbcThinSelectAfterAlterTable.class)); // Update on server - suite.addTest(new TestSuite(JdbcThinInsertStatementUpdateOnServerSelfTest.class)); - suite.addTest(new TestSuite(JdbcThinUpdateStatementUpdateOnServerSelfTest.class)); - suite.addTest(new TestSuite(JdbcThinMergeStatementUpdateOnServerSelfTest.class)); - suite.addTest(new TestSuite(JdbcThinComplexDmlDdlUpdateOnServerSelfTest.class)); + suite.addTest(new TestSuite(JdbcThinInsertStatementSkipReducerOnUpdateSelfTest.class)); + suite.addTest(new TestSuite(JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest.class)); + suite.addTest(new TestSuite(JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.class)); + suite.addTest(new TestSuite(JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.class)); return suite; diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java index 98a12e6a6943c..7f67136c202b3 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java @@ -187,7 +187,7 @@ public void testSqlHints() throws Exception { assertFalse(io(conn).collocated()); assertFalse(io(conn).replicatedOnly()); assertFalse(io(conn).lazy()); - assertFalse(io(conn).updateOnServer()); + assertFalse(io(conn).skipReducerOnUpdate()); } try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?distributedJoins=true")) { @@ -196,7 +196,7 @@ public void testSqlHints() throws Exception { assertFalse(io(conn).collocated()); assertFalse(io(conn).replicatedOnly()); assertFalse(io(conn).lazy()); - assertFalse(io(conn).updateOnServer()); + assertFalse(io(conn).skipReducerOnUpdate()); } try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?enforceJoinOrder=true")) { @@ -205,7 +205,7 @@ public void testSqlHints() throws Exception { assertFalse(io(conn).collocated()); assertFalse(io(conn).replicatedOnly()); assertFalse(io(conn).lazy()); - assertFalse(io(conn).updateOnServer()); + assertFalse(io(conn).skipReducerOnUpdate()); } try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?collocated=true")) { @@ -214,7 +214,7 @@ public void testSqlHints() throws Exception { assertTrue(io(conn).collocated()); assertFalse(io(conn).replicatedOnly()); assertFalse(io(conn).lazy()); - assertFalse(io(conn).updateOnServer()); + assertFalse(io(conn).skipReducerOnUpdate()); } try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?replicatedOnly=true")) { @@ -223,7 +223,7 @@ public void testSqlHints() throws Exception { assertFalse(io(conn).collocated()); assertTrue(io(conn).replicatedOnly()); assertFalse(io(conn).lazy()); - assertFalse(io(conn).updateOnServer()); + assertFalse(io(conn).skipReducerOnUpdate()); } try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?lazy=true")) { @@ -232,26 +232,26 @@ public void testSqlHints() throws Exception { assertFalse(io(conn).collocated()); assertFalse(io(conn).replicatedOnly()); assertTrue(io(conn).lazy()); - assertFalse(io(conn).updateOnServer()); + assertFalse(io(conn).skipReducerOnUpdate()); } - try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?updateOnServer=true")) { + try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?skipReducerOnUpdate=true")) { assertFalse(io(conn).distributedJoins()); assertFalse(io(conn).enforceJoinOrder()); assertFalse(io(conn).collocated()); assertFalse(io(conn).replicatedOnly()); assertFalse(io(conn).lazy()); - assertTrue(io(conn).updateOnServer()); + assertTrue(io(conn).skipReducerOnUpdate()); } try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?distributedJoins=true&" + - "enforceJoinOrder=true&collocated=true&replicatedOnly=true&lazy=true&updateOnServer=true")) { + "enforceJoinOrder=true&collocated=true&replicatedOnly=true&lazy=true&skipReducerOnUpdate=true")) { assertTrue(io(conn).distributedJoins()); assertTrue(io(conn).enforceJoinOrder()); assertTrue(io(conn).collocated()); assertTrue(io(conn).replicatedOnly()); assertTrue(io(conn).lazy()); - assertTrue(io(conn).updateOnServer()); + assertTrue(io(conn).skipReducerOnUpdate()); } } diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinComplexDmlDdlUpdateOnServerSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.java similarity index 89% rename from modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinComplexDmlDdlUpdateOnServerSelfTest.java rename to modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.java index 6da222f769396..06b0538a93ea6 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinComplexDmlDdlUpdateOnServerSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.java @@ -25,9 +25,9 @@ /** * Base class for complex SQL tests based on JDBC driver. */ -public class JdbcThinComplexDmlDdlUpdateOnServerSelfTest extends JdbcThinComplexDmlDdlSelfTest { +public class JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest extends JdbcThinComplexDmlDdlSelfTest { /** {@inheritDoc} */ @Override protected Connection createConnection() throws SQLException { - return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?updateOnServer=true"); + return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?skipReducerOnUpdate=true"); } } \ No newline at end of file diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinInsertStatementUpdateOnServerSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinInsertStatementSkipReducerOnUpdateSelfTest.java similarity index 89% rename from modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinInsertStatementUpdateOnServerSelfTest.java rename to modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinInsertStatementSkipReducerOnUpdateSelfTest.java index 0b4cec5579cf0..1cf32f97fccb8 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinInsertStatementUpdateOnServerSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinInsertStatementSkipReducerOnUpdateSelfTest.java @@ -25,9 +25,9 @@ /** * Statement test. */ -public class JdbcThinInsertStatementUpdateOnServerSelfTest extends JdbcThinInsertStatementSelfTest { +public class JdbcThinInsertStatementSkipReducerOnUpdateSelfTest extends JdbcThinInsertStatementSelfTest { /** {@inheritDoc} */ @Override protected Connection createConnection() throws SQLException { - return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?updateOnServer=true"); + return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?skipReducerOnUpdate=true"); } } diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinMergeStatementUpdateOnServerSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.java similarity index 89% rename from modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinMergeStatementUpdateOnServerSelfTest.java rename to modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.java index b38eca3a3c785..2a573353504c4 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinMergeStatementUpdateOnServerSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.java @@ -25,9 +25,9 @@ /** * MERGE statement test. */ -public class JdbcThinMergeStatementUpdateOnServerSelfTest extends JdbcThinMergeStatementSelfTest { +public class JdbcThinMergeStatementSkipReducerOnUpdateSelfTest extends JdbcThinMergeStatementSelfTest { /** {@inheritDoc} */ @Override protected Connection createConnection() throws SQLException { - return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?updateOnServer=true"); + return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?skipReducerOnUpdate=true"); } } diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinUpdateStatementUpdateOnServerSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest.java similarity index 89% rename from modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinUpdateStatementUpdateOnServerSelfTest.java rename to modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest.java index 9ab1fb3b7c1d4..aef7492a24dc6 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinUpdateStatementUpdateOnServerSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest.java @@ -25,9 +25,9 @@ /** * */ -public class JdbcThinUpdateStatementUpdateOnServerSelfTest extends JdbcThinUpdateStatementSelfTest { +public class JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest extends JdbcThinUpdateStatementSelfTest { /** {@inheritDoc} */ @Override protected Connection createConnection() throws SQLException { - return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?updateOnServer=true"); + return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?skipReducerOnUpdate=true"); } } diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java index 2450817931d67..ea9b7f8fa7fa3 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java @@ -334,8 +334,8 @@ public class IgniteJdbcDriver implements Driver { /** Allow queries with multiple statements. */ private static final String PARAM_MULTIPLE_STMTS = "multipleStatementsAllowed"; - /** Server side update property name. */ - private static final String PARAM_UPDATE_ON_SERVER = "updateOnServer"; + /** Skip reducer on update property name. */ + private static final String PARAM_SKIP_REDUCER_ON_UPDATE = "skipReducerOnUpdate"; /** Hostname property name. */ public static final String PROP_HOST = PROP_PREFIX + "host"; @@ -385,8 +385,8 @@ public class IgniteJdbcDriver implements Driver { /** Allow query with multiple statements. */ public static final String PROP_MULTIPLE_STMTS = PROP_PREFIX + PARAM_MULTIPLE_STMTS; - /** Server side update property name. */ - public static final String PROP_UPDATE_ON_SERVER = PROP_PREFIX + PARAM_UPDATE_ON_SERVER; + /** Skip reducer on update update property name. */ + public static final String PROP_SKIP_REDUCER_ON_UPDATE = PROP_PREFIX + PARAM_SKIP_REDUCER_ON_UPDATE; /** Cache name property name. */ public static final String PROP_CFG = PROP_PREFIX + "cfg"; @@ -461,7 +461,7 @@ public class IgniteJdbcDriver implements Driver { new JdbcDriverPropertyInfo("Lazy query execution", info.getProperty(JdbcThinUtils.PROP_LAZY), ""), new JdbcDriverPropertyInfo("Transactions Allowed", info.getProperty(PROP_TX_ALLOWED), ""), new JdbcDriverPropertyInfo("Queries with multiple statements allowed", info.getProperty(PROP_MULTIPLE_STMTS), ""), - new JdbcDriverPropertyInfo("Server side update", info.getProperty(PROP_UPDATE_ON_SERVER), "") + new JdbcDriverPropertyInfo("Skip reducer on update", info.getProperty(PROP_SKIP_REDUCER_ON_UPDATE), "") ); if (info.getProperty(PROP_CFG) != null) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcThinDriver.java b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcThinDriver.java index edf01bc8588f9..a313f92dc01f6 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcThinDriver.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcThinDriver.java @@ -187,7 +187,7 @@ public class IgniteJdbcThinDriver implements Driver { new JdbcDriverPropertyInfo("Collocated", info.getProperty(JdbcThinUtils.PROP_COLLOCATED), ""), new JdbcDriverPropertyInfo("Replicated only", info.getProperty(JdbcThinUtils.PROP_REPLICATED_ONLY), ""), new JdbcDriverPropertyInfo("Lazy query execution flag", info.getProperty(JdbcThinUtils.PROP_LAZY),""), - new JdbcDriverPropertyInfo("Execute update query on server", info.getProperty(JdbcThinUtils.PROP_UPDATE_ON_SERVER),"") + new JdbcDriverPropertyInfo("Skip reducer on update", info.getProperty(JdbcThinUtils.PROP_SKIP_REDUCER_ON_UPDATE),"") ); return props.toArray(new DriverPropertyInfo[0]); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java index 9c4b8278985d3..57b25e18360d3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java @@ -62,7 +62,7 @@ import static org.apache.ignite.internal.jdbc.thin.JdbcThinUtils.PROP_SOCK_RCV_BUF; import static org.apache.ignite.internal.jdbc.thin.JdbcThinUtils.PROP_SOCK_SND_BUF; import static org.apache.ignite.internal.jdbc.thin.JdbcThinUtils.PROP_TCP_NO_DELAY; -import static org.apache.ignite.internal.jdbc.thin.JdbcThinUtils.PROP_UPDATE_ON_SERVER; +import static org.apache.ignite.internal.jdbc.thin.JdbcThinUtils.PROP_SKIP_REDUCER_ON_UPDATE; /** * JDBC connection implementation. @@ -137,11 +137,11 @@ public JdbcThinConnection(String url, Properties props, String schema) throws SQ int sockRcvBuf = extractIntNonNegative(props, PROP_SOCK_RCV_BUF, 0); boolean tcpNoDelay = extractBoolean(props, PROP_TCP_NO_DELAY, true); - boolean updateOnServer = extractBoolean(props, PROP_UPDATE_ON_SERVER, false); + boolean skipReducerOnUpdate = extractBoolean(props, PROP_SKIP_REDUCER_ON_UPDATE, false); try { cliIo = new JdbcThinTcpIo(host, port, distributedJoins, enforceJoinOrder, collocated, replicatedOnly, - autoCloseServerCursor, lazyExec, sockSndBuf, sockRcvBuf, tcpNoDelay, updateOnServer); + autoCloseServerCursor, lazyExec, sockSndBuf, sockRcvBuf, tcpNoDelay, skipReducerOnUpdate); cliIo.start(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java index 0ffb2d46352b8..0670fb10bfc16 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java @@ -100,8 +100,8 @@ public class JdbcThinTcpIo { /** Flag to automatically close server cursor. */ private final boolean autoCloseServerCursor; - /** Executes update queries on server nodes distributedly. */ - private final boolean updateOnServer; + /** Executes update queries on server nodes. */ + private final boolean skipReducerOnUpdate; /** Socket send buffer. */ private final int sockSndBuf; @@ -141,11 +141,11 @@ public class JdbcThinTcpIo { * @param sockSndBuf Socket send buffer. * @param sockRcvBuf Socket receive buffer. * @param tcpNoDelay TCP no delay flag. - * @param updateOnServer Executes update queries on ignite server nodes distributedly. + * @param skipReducerOnUpdate Executes update queries on ignite server nodes. */ JdbcThinTcpIo(String host, int port, boolean distributedJoins, boolean enforceJoinOrder, boolean collocated, boolean replicatedOnly, boolean autoCloseServerCursor, boolean lazy, int sockSndBuf, int sockRcvBuf, - boolean tcpNoDelay, boolean updateOnServer) { + boolean tcpNoDelay, boolean skipReducerOnUpdate) { this.host = host; this.port = port; this.distributedJoins = distributedJoins; @@ -157,7 +157,7 @@ public class JdbcThinTcpIo { this.sockSndBuf = sockSndBuf; this.sockRcvBuf = sockRcvBuf; this.tcpNoDelay = tcpNoDelay; - this.updateOnServer = updateOnServer; + this.skipReducerOnUpdate = skipReducerOnUpdate; } /** @@ -216,7 +216,7 @@ public void handshake(ClientListenerProtocolVersion ver) throws IOException, SQL writer.writeBoolean(replicatedOnly); writer.writeBoolean(autoCloseServerCursor); writer.writeBoolean(lazy); - writer.writeBoolean(updateOnServer); + writer.writeBoolean(skipReducerOnUpdate); send(writer.array()); @@ -501,7 +501,7 @@ public boolean lazy() { /** * @return Server side update flag. */ - public boolean updateOnServer() { - return updateOnServer; + public boolean skipReducerOnUpdate() { + return skipReducerOnUpdate; } } \ No newline at end of file diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinUtils.java index 12fe25face637..c9bf61cab5583 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinUtils.java @@ -82,7 +82,7 @@ public class JdbcThinUtils { public static final String PARAM_AUTO_CLOSE_SERVER_CURSOR = "autoCloseServerCursor"; /** Parameter: execute update query in distributed mode on ignite server nodes. */ - public static final String PARAM_UPDATE_ON_SERVER = "updateOnServer"; + public static final String PARAM_SKIP_REDUCER_ON_UPDATE = "skipReducerOnUpdate"; /** Distributed joins property name. */ public static final String PROP_DISTRIBUTED_JOINS = PROP_PREFIX + PARAM_DISTRIBUTED_JOINS; @@ -112,7 +112,7 @@ public class JdbcThinUtils { public static final String PROP_AUTO_CLOSE_SERVER_CURSORS = PROP_PREFIX + PARAM_AUTO_CLOSE_SERVER_CURSOR; /** Executes update queries on ignite server nodes in distributed mode. */ - public static final String PROP_UPDATE_ON_SERVER = PROP_PREFIX + PARAM_UPDATE_ON_SERVER; + public static final String PROP_SKIP_REDUCER_ON_UPDATE = PROP_PREFIX + PARAM_SKIP_REDUCER_ON_UPDATE; /** Default port. */ public static final int DFLT_PORT = ClientConnectorConfiguration.DFLT_PORT; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java index 4e48ae5684191..29cb6a1669dea 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java @@ -88,7 +88,7 @@ import static org.apache.ignite.IgniteJdbcDriver.PROP_STREAMING_PER_NODE_BUF_SIZE; import static org.apache.ignite.IgniteJdbcDriver.PROP_STREAMING_PER_NODE_PAR_OPS; import static org.apache.ignite.IgniteJdbcDriver.PROP_TX_ALLOWED; -import static org.apache.ignite.IgniteJdbcDriver.PROP_UPDATE_ON_SERVER; +import static org.apache.ignite.IgniteJdbcDriver.PROP_SKIP_REDUCER_ON_UPDATE; import static org.apache.ignite.internal.jdbc2.JdbcUtils.convertToSqlException; import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.createJdbcSqlException; @@ -169,8 +169,8 @@ public class JdbcConnection implements Connection { /** Allow queries with multiple statements. */ private final boolean multipleStmts; - /** Update on server flag. */ - private final boolean updateOnServer; + /** Skip reducer on update flag. */ + private final boolean skipReducerOnUpdate; /** Statements. */ final Set statements = new HashSet<>(); @@ -213,7 +213,7 @@ public JdbcConnection(String url, Properties props) throws SQLException { streamNodeParOps = Integer.parseInt(props.getProperty(PROP_STREAMING_PER_NODE_PAR_OPS, "0")); multipleStmts = Boolean.parseBoolean(props.getProperty(PROP_MULTIPLE_STMTS)); - updateOnServer = Boolean.parseBoolean(props.getProperty(PROP_UPDATE_ON_SERVER)); + skipReducerOnUpdate = Boolean.parseBoolean(props.getProperty(PROP_SKIP_REDUCER_ON_UPDATE)); String nodeIdProp = props.getProperty(PROP_NODE_ID); @@ -861,8 +861,8 @@ boolean isMultipleStatementsAllowed() { /** * @return {@code true} if update on server is enabled, {@code false} otherwise. */ - boolean updateOnServer() { - return updateOnServer; + boolean skipReducerOnUpdate() { + return skipReducerOnUpdate; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java index d97b833c81da6..aa9f009962db2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTask.java @@ -244,7 +244,7 @@ protected boolean updateMetadata() { /** * @return Flag to update enable server side updates. */ - protected boolean updateOnServer() { + protected boolean skipReducerOnUpdate() { return false; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV3.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV3.java index 986b3819dde19..f002d87127825 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV3.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcQueryTaskV3.java @@ -31,7 +31,7 @@ class JdbcQueryTaskV3 extends JdbcQueryTaskV2 { private final boolean updateMeta; /** Update metadata on demand flag. */ - private final boolean updateOnServer; + private final boolean skipReducerOnUpdate; /** * @param ignite Ignite. @@ -49,16 +49,16 @@ class JdbcQueryTaskV3 extends JdbcQueryTaskV2 { * @param enforceJoinOrder Enforce joins order flag. * @param lazy Lazy query execution flag. * @param updateMeta Update metadata on demand. - * @param updateOnServer Flkag to enable server side updates. + * @param skipReducerOnUpdate Flkag to enable server side updates. */ public JdbcQueryTaskV3(Ignite ignite, String cacheName, String schemaName, String sql, Boolean isQry, boolean loc, Object[] args, int fetchSize, UUID uuid, boolean locQry, boolean collocatedQry, boolean distributedJoins, - boolean enforceJoinOrder, boolean lazy, boolean updateMeta, boolean updateOnServer) { + boolean enforceJoinOrder, boolean lazy, boolean updateMeta, boolean skipReducerOnUpdate) { super(ignite, cacheName, schemaName, sql, isQry, loc, args, fetchSize, uuid, locQry, collocatedQry, distributedJoins, enforceJoinOrder, lazy); this.updateMeta = updateMeta; - this.updateOnServer = updateOnServer; + this.skipReducerOnUpdate = skipReducerOnUpdate; } /** {@inheritDoc} */ @@ -67,8 +67,8 @@ public JdbcQueryTaskV3(Ignite ignite, String cacheName, String schemaName, Strin } /** {@inheritDoc} */ - @Override protected boolean updateOnServer() { - return updateOnServer; + @Override protected boolean skipReducerOnUpdate() { + return skipReducerOnUpdate; } /** @@ -87,17 +87,17 @@ public JdbcQueryTaskV3(Ignite ignite, String cacheName, String schemaName, Strin * @param enforceJoinOrder Enforce joins order flag. * @param lazy Lazy query execution flag. * @param updateMeta Update metadata on demand. - * @param updateOnServer Update on server flag. + * @param skipReducerOnUpdate Update on server flag. * @return Appropriate task JdbcQueryTask or JdbcQueryTaskV2. */ public static JdbcQueryTask createTask(Ignite ignite, String cacheName, String schemaName, String sql, Boolean isQry, boolean loc, Object[] args, int fetchSize, UUID uuid, boolean locQry, boolean collocatedQry, boolean distributedJoins, - boolean enforceJoinOrder, boolean lazy, boolean updateMeta, boolean updateOnServer) { + boolean enforceJoinOrder, boolean lazy, boolean updateMeta, boolean skipReducerOnUpdate) { - if (updateMeta || updateOnServer) + if (updateMeta || skipReducerOnUpdate) return new JdbcQueryTaskV3(ignite, cacheName, schemaName, sql, isQry, loc, args, fetchSize, - uuid, locQry, collocatedQry, distributedJoins, enforceJoinOrder, lazy, updateMeta, updateOnServer); + uuid, locQry, collocatedQry, distributedJoins, enforceJoinOrder, lazy, updateMeta, skipReducerOnUpdate); else return JdbcQueryTaskV2.createTask(ignite, cacheName, schemaName, sql, isQry, loc, args, fetchSize, uuid, locQry, collocatedQry, distributedJoins, enforceJoinOrder, lazy); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java index b1798d8f91b55..2498456be53eb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java @@ -163,7 +163,7 @@ private void executeSingle(String sql, Boolean isQuery) throws SQLException { JdbcQueryTask qryTask = JdbcQueryTaskV3.createTask(loc ? ignite : null, conn.cacheName(), conn.schemaName(), sql, isQuery, loc, getArgs(), fetchSize, uuid, conn.isLocalQuery(), conn.isCollocatedQuery(), - conn.isDistributedJoins(), conn.isEnforceJoinOrder(), conn.isLazy(), false, conn.updateOnServer()); + conn.isDistributedJoins(), conn.isEnforceJoinOrder(), conn.isLazy(), false, conn.skipReducerOnUpdate()); try { JdbcQueryTaskResult qryRes = diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java index 3a7c6a6dab052..ef8f2783de46d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java @@ -31,7 +31,7 @@ public final class SqlFieldsQueryEx extends SqlFieldsQuery { private final Boolean isQry; /** Whether server side DML should be enabled. */ - private boolean updateOnServer; + private boolean skipReducerOnUpdate; /** * @param sql SQL query. @@ -49,7 +49,7 @@ private SqlFieldsQueryEx(SqlFieldsQueryEx qry) { super(qry); this.isQry = qry.isQry; - this.updateOnServer = qry.updateOnServer; + this.skipReducerOnUpdate = qry.skipReducerOnUpdate; } /** @@ -134,8 +134,8 @@ public Boolean isQuery() { * @param updateOnServer Server side update flag. * @return {@code this} For chaining. */ - public SqlFieldsQuery setUpdateOnServer(boolean updateOnServer) { - this.updateOnServer = updateOnServer; + public SqlFieldsQuery setSkipReducerOnUpdate(boolean updateOnServer) { + this.skipReducerOnUpdate = updateOnServer; return this; } @@ -143,12 +143,12 @@ public SqlFieldsQuery setUpdateOnServer(boolean updateOnServer) { /** * Gets server side update flag. *

- * See {@link #setUpdateOnServer(boolean)} for more information. + * See {@link #setSkipReducerOnUpdate(boolean)} for more information. * * @return Server side update flag. */ - public boolean isUpdateOnServer() { - return updateOnServer; + public boolean isSkipReducerOnUpdate() { + return skipReducerOnUpdate; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java index da71be6510a07..7b404664e33e2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcConnectionContext.java @@ -104,13 +104,13 @@ public JdbcConnectionContext(GridKernalContext ctx, GridSpinBusyLock busyLock, i if (ver.compareTo(VER_2_1_5) >= 0) lazyExec = reader.readBoolean(); - boolean updateOnServer = false; + boolean skipReducerOnUpdate = false; if (ver.compareTo(VER_2_3_0) >= 0) - updateOnServer = reader.readBoolean(); + skipReducerOnUpdate = reader.readBoolean(); handler = new JdbcRequestHandler(ctx, busyLock, maxCursors, distributedJoins, enforceJoinOrder, - collocated, replicatedOnly, autoCloseCursors, lazyExec, updateOnServer, ver); + collocated, replicatedOnly, autoCloseCursors, lazyExec, skipReducerOnUpdate, ver); parser = new JdbcMessageParser(ctx); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java index add9ab45a6efb..e3b6f5b5ef80e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/jdbc/JdbcRequestHandler.java @@ -103,8 +103,8 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler { /** Lazy query execution flag. */ private final boolean lazy; - /** Update on server flag. */ - private final boolean updateOnServer; + /** Skip reducer on update flag. */ + private final boolean skipReducerOnUpdate; /** Automatic close of cursors. */ private final boolean autoCloseCursors; @@ -124,12 +124,13 @@ public class JdbcRequestHandler implements ClientListenerRequestHandler { * @param replicatedOnly Replicated only flag. * @param autoCloseCursors Flag to automatically close server cursors. * @param lazy Lazy query execution flag. - * @param updateOnServer Server side update flag. + * @param skipReducerOnUpdate Skip reducer on update flag. * @param protocolVer Protocol version. */ public JdbcRequestHandler(GridKernalContext ctx, GridSpinBusyLock busyLock, int maxCursors, boolean distributedJoins, boolean enforceJoinOrder, boolean collocated, boolean replicatedOnly, - boolean autoCloseCursors, boolean lazy, boolean updateOnServer, ClientListenerProtocolVersion protocolVer) { + boolean autoCloseCursors, boolean lazy, boolean skipReducerOnUpdate, + ClientListenerProtocolVersion protocolVer) { this.ctx = ctx; this.busyLock = busyLock; this.maxCursors = maxCursors; @@ -139,7 +140,7 @@ public JdbcRequestHandler(GridKernalContext ctx, GridSpinBusyLock busyLock, int this.replicatedOnly = replicatedOnly; this.autoCloseCursors = autoCloseCursors; this.lazy = lazy; - this.updateOnServer = updateOnServer; + this.skipReducerOnUpdate = skipReducerOnUpdate; this.protocolVer = protocolVer; log = ctx.log(getClass()); @@ -277,8 +278,8 @@ private JdbcResponse executeQuery(JdbcQueryExecuteRequest req) { qry = new SqlFieldsQueryEx(sql, false); - if (updateOnServer) - ((SqlFieldsQueryEx)qry).setUpdateOnServer(true); + if (skipReducerOnUpdate) + ((SqlFieldsQueryEx)qry).setSkipReducerOnUpdate(true); } qry.setArgs(req.arguments()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java index 298bc12244420..a42e7801165ec 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java @@ -102,13 +102,13 @@ public OdbcConnectionContext(GridKernalContext ctx, GridSpinBusyLock busyLock, i if (ver.compareTo(VER_2_1_5) >= 0) lazy = reader.readBoolean(); - boolean updateOnServer = false; + boolean skipReducerOnUpdate = false; if (ver.compareTo(VER_2_3_0) >= 0) - updateOnServer = reader.readBoolean(); + skipReducerOnUpdate = reader.readBoolean(); handler = new OdbcRequestHandler(ctx, busyLock, maxCursors, distributedJoins, - enforceJoinOrder, replicatedOnly, collocated, lazy, updateOnServer); + enforceJoinOrder, replicatedOnly, collocated, lazy, skipReducerOnUpdate); parser = new OdbcMessageParser(ctx, ver); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java index f4564a4610bdc..32375fddd7a06 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java @@ -95,7 +95,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler { private final boolean lazy; /** Update on server flag. */ - private final boolean updateOnServer; + private final boolean skipReducerOnUpdate; /** * Constructor. @@ -107,11 +107,11 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler { * @param replicatedOnly Replicated only flag. * @param collocated Collocated flag. * @param lazy Lazy flag. - * @param updateOnServer Update on server flag. + * @param skipReducerOnUpdate Skip reducer on update flag. */ public OdbcRequestHandler(GridKernalContext ctx, GridSpinBusyLock busyLock, int maxCursors, boolean distributedJoins, boolean enforceJoinOrder, boolean replicatedOnly, - boolean collocated, boolean lazy, boolean updateOnServer) { + boolean collocated, boolean lazy, boolean skipReducerOnUpdate) { this.ctx = ctx; this.busyLock = busyLock; this.maxCursors = maxCursors; @@ -120,7 +120,7 @@ public OdbcRequestHandler(GridKernalContext ctx, GridSpinBusyLock busyLock, int this.replicatedOnly = replicatedOnly; this.collocated = collocated; this.lazy = lazy; - this.updateOnServer = updateOnServer; + this.skipReducerOnUpdate = skipReducerOnUpdate; log = ctx.log(getClass()); } @@ -212,7 +212,7 @@ private SqlFieldsQueryEx makeQuery(String schema, String sql, Object[] args) { qry.setCollocated(collocated); qry.setLazy(lazy); qry.setSchema(schema); - qry.setUpdateOnServer(updateOnServer); + qry.setSkipReducerOnUpdate(skipReducerOnUpdate); return qry; } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java index ead4629ba4cf6..11e4e281f956e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java @@ -587,7 +587,7 @@ private static UpdatePlan.DistributedPlanInfo checkPlanCanBeDistributed(IgniteH2 */ public static boolean isUpdateOnServerQuery(SqlFieldsQuery qry) { return qry != null && !qry.isLocal() && - qry instanceof SqlFieldsQueryEx && ((SqlFieldsQueryEx)qry).isUpdateOnServer(); + qry instanceof SqlFieldsQueryEx && ((SqlFieldsQueryEx)qry).isSkipReducerOnUpdate(); } /** diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java index 3d00e3afc99d7..56d044e54fc63 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java @@ -586,7 +586,7 @@ private void checkUpdate(IgniteCache cache, Map initial, SqlF if (!F.isEmpty(initial)) cache.putAll(initial); - List> updRes = cache.query(qry.setUpdateOnServer(true)).getAll(); + List> updRes = cache.query(qry.setSkipReducerOnUpdate(true)).getAll(); Map result = new HashMap<>(cache.size()); @@ -598,7 +598,7 @@ private void checkUpdate(IgniteCache cache, Map initial, SqlF if (!F.isEmpty(initial)) cache.putAll(initial); - List> updRes2 = cache.query(qry.setUpdateOnServer(false)).getAll(); + List> updRes2 = cache.query(qry.setSkipReducerOnUpdate(false)).getAll(); assertTrue(((Number)updRes.get(0).get(0)).intValue() > 0); diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java index 1113dd511cacb..bf5c645d36604 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java @@ -211,7 +211,7 @@ public void testSimpleUpdateDistributedReplicated() throws Exception { Position p = cache.get(1); List> r = cache.query(new SqlFieldsQueryEx("UPDATE Position p SET name = CONCAT('A ', name)", false) - .setUpdateOnServer(true)).getAll(); + .setSkipReducerOnUpdate(true)).getAll(); assertEquals((long)cache.size(), r.get(0).get(0)); @@ -229,7 +229,7 @@ public void testSimpleUpdateDistributedPartitioned() throws Exception { List> r = cache.query(new SqlFieldsQueryEx( "UPDATE Person SET position = CASEWHEN(position = 1, 1, position - 1)", false) - .setUpdateOnServer(true)).getAll(); + .setSkipReducerOnUpdate(true)).getAll(); assertEquals((long)cache.size(), r.get(0).get(0)); } @@ -247,7 +247,7 @@ public void testDistributedUpdateFailedKeys() throws Exception { GridTestUtils.assertThrows(log, new Callable() { @Override public Object call() { return cache.query(new SqlFieldsQueryEx("UPDATE Organization SET rate = Modify(_key, rate - 1)", false) - .setUpdateOnServer(true)); + .setSkipReducerOnUpdate(true)); } }, CacheException.class, "Failed to update some keys because they had been modified concurrently"); } @@ -264,7 +264,7 @@ public void testDistributedUpdateFail() throws Exception { GridTestUtils.assertThrows(log, new Callable() { @Override public Object call() { return cache.query(new SqlFieldsQueryEx("UPDATE Person SET name = Fail(name)", false) - .setUpdateOnServer(true)); + .setSkipReducerOnUpdate(true)); } }, CacheException.class, "Failed to execute SQL query"); } @@ -287,7 +287,7 @@ public void testQueryParallelism() throws Exception { cache.put(i, new Organization("Acme Inc #" + i, 0)); List> r = cache.query(new SqlFieldsQueryEx("UPDATE \"" + cacheName + - "\".Organization o SET name = UPPER(name)", false).setUpdateOnServer(true)).getAll(); + "\".Organization o SET name = UPPER(name)", false).setSkipReducerOnUpdate(true)).getAll(); assertEquals((long)cache.size(), r.get(0).get(0)); } @@ -322,7 +322,7 @@ public void testEvents() throws Exception { cache.put(i, new Organization("Acme Inc #" + i, 0)); cache.query(new SqlFieldsQueryEx("UPDATE \"org\".Organization o SET name = UPPER(name)", false) - .setUpdateOnServer(true)).getAll(); + .setSkipReducerOnUpdate(true)).getAll(); assertTrue(latch.await(5000, MILLISECONDS)); @@ -349,7 +349,7 @@ public void testSpecificPartitionsUpdate() throws Exception { // UPDATE over even partitions cache.query(new SqlFieldsQueryEx("UPDATE Person SET position = 0", false) - .setUpdateOnServer(true) + .setSkipReducerOnUpdate(true) .setPartitions(parts)); List> rows = cache.query(new SqlFieldsQuery("SELECT _key, position FROM Person")).getAll(); @@ -377,7 +377,7 @@ public void testCancel() throws Exception { final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { @Override public Object call() { return cache.query(new SqlFieldsQueryEx("UPDATE Organization SET name = WAIT(name)", false) - .setUpdateOnServer(true)); + .setSkipReducerOnUpdate(true)); } }); @@ -423,7 +423,7 @@ public void testNodeStopDuringUpdate() throws Exception { final IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() { @Override public Object call() { return cache.query(new SqlFieldsQueryEx("UPDATE Organization SET name = WAIT(name)", false) - .setUpdateOnServer(true)); + .setSkipReducerOnUpdate(true)); } }); diff --git a/modules/platforms/cpp/odbc-test/src/configuration_test.cpp b/modules/platforms/cpp/odbc-test/src/configuration_test.cpp index 3a331144bd282..3165c4d268b3c 100644 --- a/modules/platforms/cpp/odbc-test/src/configuration_test.cpp +++ b/modules/platforms/cpp/odbc-test/src/configuration_test.cpp @@ -43,7 +43,7 @@ namespace const bool testReplicatedOnly = true; const bool testCollocated = true; const bool testLazy = true; - const bool testUpdateOnServer = true; + const bool testSkipReducerOnUpdate = true; const std::string testAddress = testServerHost + ':' + ignite::common::LexicalCast(testServerPort); } @@ -133,7 +133,7 @@ void CheckConnectionConfig(const Configuration& cfg) BOOST_CHECK_EQUAL(cfg.IsReplicatedOnly(), testReplicatedOnly); BOOST_CHECK_EQUAL(cfg.IsCollocated(), testCollocated); BOOST_CHECK_EQUAL(cfg.IsLazy(), testLazy); - BOOST_CHECK_EQUAL(cfg.IsUpdateOnServer(), testUpdateOnServer); + BOOST_CHECK_EQUAL(cfg.IsSkipReducerOnUpdate(), testSkipReducerOnUpdate); std::stringstream constructor; @@ -146,7 +146,7 @@ void CheckConnectionConfig(const Configuration& cfg) << "page_size=" << testPageSize << ';' << "replicated_only=" << BoolToStr(testReplicatedOnly) << ';' << "schema=" << testSchemaName << ';' - << "update_on_server=" << BoolToStr(testReplicatedOnly) << ';'; + << "skip_reducer_on_update=" << BoolToStr(testReplicatedOnly) << ';'; const std::string& expectedStr = constructor.str(); @@ -167,7 +167,7 @@ void CheckDsnConfig(const Configuration& cfg) BOOST_CHECK_EQUAL(cfg.IsReplicatedOnly(), false); BOOST_CHECK_EQUAL(cfg.IsCollocated(), false); BOOST_CHECK_EQUAL(cfg.IsLazy(), false); - BOOST_CHECK_EQUAL(cfg.IsUpdateOnServer(), false); + BOOST_CHECK_EQUAL(cfg.IsSkipReducerOnUpdate(), false); } BOOST_AUTO_TEST_SUITE(ConfigurationTestSuite) @@ -185,7 +185,7 @@ BOOST_AUTO_TEST_CASE(CheckTestValuesNotEquealDefault) BOOST_CHECK_NE(testReplicatedOnly, Configuration::DefaultValue::replicatedOnly); BOOST_CHECK_NE(testCollocated, Configuration::DefaultValue::collocated); BOOST_CHECK_NE(testLazy, Configuration::DefaultValue::lazy); - BOOST_CHECK_NE(testUpdateOnServer, Configuration::DefaultValue::updateOnServer); + BOOST_CHECK_NE(testSkipReducerOnUpdate, Configuration::DefaultValue::skipReducerOnUpdate); } BOOST_AUTO_TEST_CASE(TestConnectStringUppercase) @@ -203,7 +203,7 @@ BOOST_AUTO_TEST_CASE(TestConnectStringUppercase) << "REPLICATED_ONLY=" << BoolToStr(testReplicatedOnly, false) << ';' << "PAGE_SIZE=" << testPageSize << ';' << "SCHEMA=" << testSchemaName << ';' - << "UPDATE_ON_SERVER=" << BoolToStr(testUpdateOnServer, false); + << "SKIP_REDUCER_ON_UPDATE=" << BoolToStr(testSkipReducerOnUpdate, false); const std::string& connectStr = constructor.str(); @@ -227,7 +227,7 @@ BOOST_AUTO_TEST_CASE(TestConnectStringLowercase) << "replicated_only=" << BoolToStr(testReplicatedOnly) << ';' << "collocated=" << BoolToStr(testCollocated) << ';' << "schema=" << testSchemaName << ';' - << "update_on_server=" << BoolToStr(testUpdateOnServer); + << "skip_reducer_on_update=" << BoolToStr(testSkipReducerOnUpdate); const std::string& connectStr = constructor.str(); @@ -251,7 +251,7 @@ BOOST_AUTO_TEST_CASE(TestConnectStringZeroTerminated) << "distributed_joins=" << BoolToStr(testDistributedJoins) << ';' << "enforce_join_order=" << BoolToStr(testEnforceJoinOrder) << ';' << "schema=" << testSchemaName << ';' - << "update_on_server=" << BoolToStr(testUpdateOnServer); + << "skip_reducer_on_update=" << BoolToStr(testSkipReducerOnUpdate); const std::string& connectStr = constructor.str(); @@ -275,7 +275,7 @@ BOOST_AUTO_TEST_CASE(TestConnectStringMixed) << "Replicated_Only=" << BoolToStr(testReplicatedOnly, false) << ';' << "Collocated=" << BoolToStr(testCollocated) << ';' << "Schema=" << testSchemaName << ';' - << "Update_On_Server=" << BoolToStr(testUpdateOnServer); + << "Skip_Reducer_On_Update=" << BoolToStr(testSkipReducerOnUpdate); const std::string& connectStr = constructor.str(); @@ -299,7 +299,7 @@ BOOST_AUTO_TEST_CASE(TestConnectStringWhitepaces) << " REPLICATED_ONLY= " << BoolToStr(testReplicatedOnly, false) << ';' << "ENFORCE_JOIN_ORDER= " << BoolToStr(testEnforceJoinOrder, false) << " ;" << "SCHEMA = \n\r" << testSchemaName << ';' - << " update_on_server=" << BoolToStr(testUpdateOnServer, false); + << " skip_reducer_on_update=" << BoolToStr(testSkipReducerOnUpdate, false); const std::string& connectStr = constructor.str(); @@ -369,7 +369,7 @@ BOOST_AUTO_TEST_CASE(TestConnectStringInvalidBoolKeys) keys.insert("replicated_only"); keys.insert("collocated"); keys.insert("lazy"); - keys.insert("update_on_server"); + keys.insert("skip_reducer_on_update"); for (Set::const_iterator it = keys.begin(); it != keys.end(); ++it) { @@ -397,7 +397,7 @@ BOOST_AUTO_TEST_CASE(TestConnectStringValidBoolKeys) keys.insert("replicated_only"); keys.insert("collocated"); keys.insert("lazy"); - keys.insert("update_on_server"); + keys.insert("skip_reducer_on_update"); for (Set::const_iterator it = keys.begin(); it != keys.end(); ++it) { diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/config/configuration.h b/modules/platforms/cpp/odbc/include/ignite/odbc/config/configuration.h index d675f82076aad..419a65e72a09b 100644 --- a/modules/platforms/cpp/odbc/include/ignite/odbc/config/configuration.h +++ b/modules/platforms/cpp/odbc/include/ignite/odbc/config/configuration.h @@ -83,8 +83,8 @@ namespace ignite /** Connection attribute keyword for lazy attribute. */ static const std::string lazy; - /** Connection attribute keyword for updateOnServer attribute. */ - static const std::string updateOnServer; + /** Connection attribute keyword for skipReducerOnUpdate attribute. */ + static const std::string skipReducerOnUpdate; }; /** Default values for configuration. */ @@ -129,8 +129,8 @@ namespace ignite /** Default value for lazy attribute. */ static const bool lazy; - /** Default value for updateOnServer attribute. */ - static const bool updateOnServer; + /** Default value for skipReducerOnUpdate attribute. */ + static const bool skipReducerOnUpdate; }; /** @@ -394,9 +394,9 @@ namespace ignite * * @return True if update on server. */ - bool IsUpdateOnServer() const + bool IsSkipReducerOnUpdate() const { - return GetBoolValue(Key::updateOnServer, DefaultValue::updateOnServer); + return GetBoolValue(Key::skipReducerOnUpdate, DefaultValue::skipReducerOnUpdate); } /** @@ -404,9 +404,9 @@ namespace ignite * * @param val Value to set. */ - void SetUpdateOnServer(bool val) + void SetSkipReducerOnUpdate(bool val) { - SetBoolValue(Key::updateOnServer, val); + SetBoolValue(Key::skipReducerOnUpdate, val); } /** diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h index d88a16a7817ff..dda0ba93e8558 100644 --- a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h +++ b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h @@ -79,10 +79,10 @@ namespace ignite * @param replicatedOnly Replicated only flag. * @param collocated Collocated flag. * @param lazy Lazy flag. - * @param updateOnServer Update on server. + * @param skipReducerOnUpdate Skip reducer on update. */ HandshakeRequest(const ProtocolVersion& version, bool distributedJoins, bool enforceJoinOrder, - bool replicatedOnly, bool collocated, bool lazy, bool updateOnServer); + bool replicatedOnly, bool collocated, bool lazy, bool skipReducerOnUpdate); /** * Destructor. @@ -114,8 +114,8 @@ namespace ignite /** Lazy flag. */ bool lazy; - /** Update on server flag. */ - bool updateOnServer; + /** Skip reducer on update flag. */ + bool skipReducerOnUpdate; }; /** diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/system/ui/dsn_configuration_window.h b/modules/platforms/cpp/odbc/include/ignite/odbc/system/ui/dsn_configuration_window.h index 7b9be8718aa80..90286b941fcb7 100644 --- a/modules/platforms/cpp/odbc/include/ignite/odbc/system/ui/dsn_configuration_window.h +++ b/modules/platforms/cpp/odbc/include/ignite/odbc/system/ui/dsn_configuration_window.h @@ -55,7 +55,7 @@ namespace ignite REPLICATED_ONLY_CHECK_BOX, COLLOCATED_CHECK_BOX, LAZY_CHECK_BOX, - UPDATE_ON_SERVER_CHECK_BOX, + SKIP_REDUCER_ON_UPDATE_CHECK_BOX, PROTOCOL_VERSION_LABEL, PROTOCOL_VERSION_COMBO_BOX, OK_BUTTON, @@ -151,7 +151,7 @@ namespace ignite std::auto_ptr lazyCheckBox; /** Update on server CheckBox. */ - std::auto_ptr updateOnServerCheckBox; + std::auto_ptr skipReducerOnUpdateCheckBox; /** Protocol version edit field. */ std::auto_ptr protocolVersionLabel; diff --git a/modules/platforms/cpp/odbc/os/win/src/system/ui/dsn_configuration_window.cpp b/modules/platforms/cpp/odbc/os/win/src/system/ui/dsn_configuration_window.cpp index 40f7b5ffe74dc..d5aa0dbd723b8 100644 --- a/modules/platforms/cpp/odbc/os/win/src/system/ui/dsn_configuration_window.cpp +++ b/modules/platforms/cpp/odbc/os/win/src/system/ui/dsn_configuration_window.cpp @@ -180,10 +180,11 @@ namespace ignite lazyCheckBox->SetEnabled(version >= ProtocolVersion::VERSION_2_1_5); - updateOnServerCheckBox = CreateCheckBox(editPosX + checkBoxSize + interval, rowPos, checkBoxSize, - rowSize, "Update on server", ChildId::UPDATE_ON_SERVER_CHECK_BOX, config.IsUpdateOnServer()); + skipReducerOnUpdateCheckBox = CreateCheckBox(editPosX + checkBoxSize + interval, rowPos, + checkBoxSize, rowSize, "Skip reducer on update", ChildId::SKIP_REDUCER_ON_UPDATE_CHECK_BOX, + config.IsSkipReducerOnUpdate()); - updateOnServerCheckBox->SetEnabled(version >= ProtocolVersion::VERSION_2_3_0); + skipReducerOnUpdateCheckBox->SetEnabled(version >= ProtocolVersion::VERSION_2_3_0); rowPos += interval * 2 + rowSize; @@ -269,9 +270,9 @@ namespace ignite break; } - case ChildId::UPDATE_ON_SERVER_CHECK_BOX: + case ChildId::SKIP_REDUCER_ON_UPDATE_CHECK_BOX: { - updateOnServerCheckBox->SetChecked(!updateOnServerCheckBox->IsChecked()); + skipReducerOnUpdateCheckBox->SetChecked(!skipReducerOnUpdateCheckBox->IsChecked()); break; } @@ -283,7 +284,7 @@ namespace ignite ProtocolVersion version = ProtocolVersion::FromString(versionStr); lazyCheckBox->SetEnabled(version >= ProtocolVersion::VERSION_2_1_5); - updateOnServerCheckBox->SetEnabled(version >= ProtocolVersion::VERSION_2_3_0); + skipReducerOnUpdateCheckBox->SetEnabled(version >= ProtocolVersion::VERSION_2_3_0); break; } @@ -322,7 +323,7 @@ namespace ignite bool replicatedOnly; bool collocated; bool lazy; - bool updateOnServer; + bool skipReducerOnUpdate; nameEdit->GetText(dsn); addressEdit->GetText(address); @@ -343,7 +344,9 @@ namespace ignite replicatedOnly = replicatedOnlyCheckBox->IsEnabled() && replicatedOnlyCheckBox->IsChecked(); collocated = collocatedCheckBox->IsEnabled() && collocatedCheckBox->IsChecked(); lazy = lazyCheckBox->IsEnabled() && lazyCheckBox->IsChecked(); - updateOnServer = updateOnServerCheckBox->IsEnabled() && updateOnServerCheckBox->IsChecked(); + + skipReducerOnUpdate = + skipReducerOnUpdateCheckBox->IsEnabled() && skipReducerOnUpdateCheckBox->IsChecked(); LOG_MSG("Retriving arguments:"); LOG_MSG("DSN: " << dsn); @@ -356,7 +359,7 @@ namespace ignite LOG_MSG("Replicated only: " << (replicatedOnly ? "true" : "false")); LOG_MSG("Collocated: " << (collocated ? "true" : "false")); LOG_MSG("Lazy: " << (lazy ? "true" : "false")); - LOG_MSG("Update on server: " << (updateOnServer ? "true" : "false")); + LOG_MSG("Skip reducer on update: " << (skipReducerOnUpdate ? "true" : "false")); if (dsn.empty()) throw IgniteError(IgniteError::IGNITE_ERR_GENERIC, "DSN name can not be empty."); @@ -371,7 +374,7 @@ namespace ignite cfg.SetReplicatedOnly(replicatedOnly); cfg.SetCollocated(collocated); cfg.SetLazy(lazy); - cfg.SetUpdateOnServer(updateOnServer); + cfg.SetSkipReducerOnUpdate(skipReducerOnUpdate); } } } diff --git a/modules/platforms/cpp/odbc/src/config/configuration.cpp b/modules/platforms/cpp/odbc/src/config/configuration.cpp index 6bb1ab6b95c6f..be5a7814fb46d 100644 --- a/modules/platforms/cpp/odbc/src/config/configuration.cpp +++ b/modules/platforms/cpp/odbc/src/config/configuration.cpp @@ -32,36 +32,36 @@ namespace ignite { namespace config { - const std::string Configuration::Key::dsn = "dsn"; - const std::string Configuration::Key::driver = "driver"; - const std::string Configuration::Key::schema = "schema"; - const std::string Configuration::Key::address = "address"; - const std::string Configuration::Key::server = "server"; - const std::string Configuration::Key::port = "port"; - const std::string Configuration::Key::distributedJoins = "distributed_joins"; - const std::string Configuration::Key::enforceJoinOrder = "enforce_join_order"; - const std::string Configuration::Key::protocolVersion = "protocol_version"; - const std::string Configuration::Key::pageSize = "page_size"; - const std::string Configuration::Key::replicatedOnly = "replicated_only"; - const std::string Configuration::Key::collocated = "collocated"; - const std::string Configuration::Key::lazy = "lazy"; - const std::string Configuration::Key::updateOnServer = "update_on_server"; - - const std::string Configuration::DefaultValue::dsn = "Apache Ignite DSN"; - const std::string Configuration::DefaultValue::driver = "Apache Ignite"; - const std::string Configuration::DefaultValue::schema = "PUBLIC"; - const std::string Configuration::DefaultValue::address = ""; - const std::string Configuration::DefaultValue::server = ""; + const std::string Configuration::Key::dsn = "dsn"; + const std::string Configuration::Key::driver = "driver"; + const std::string Configuration::Key::schema = "schema"; + const std::string Configuration::Key::address = "address"; + const std::string Configuration::Key::server = "server"; + const std::string Configuration::Key::port = "port"; + const std::string Configuration::Key::distributedJoins = "distributed_joins"; + const std::string Configuration::Key::enforceJoinOrder = "enforce_join_order"; + const std::string Configuration::Key::protocolVersion = "protocol_version"; + const std::string Configuration::Key::pageSize = "page_size"; + const std::string Configuration::Key::replicatedOnly = "replicated_only"; + const std::string Configuration::Key::collocated = "collocated"; + const std::string Configuration::Key::lazy = "lazy"; + const std::string Configuration::Key::skipReducerOnUpdate = "skip_reducer_on_update"; + + const std::string Configuration::DefaultValue::dsn = "Apache Ignite DSN"; + const std::string Configuration::DefaultValue::driver = "Apache Ignite"; + const std::string Configuration::DefaultValue::schema = "PUBLIC"; + const std::string Configuration::DefaultValue::address = ""; + const std::string Configuration::DefaultValue::server = ""; const uint16_t Configuration::DefaultValue::port = 10800; const int32_t Configuration::DefaultValue::pageSize = 1024; - const bool Configuration::DefaultValue::distributedJoins = false; - const bool Configuration::DefaultValue::enforceJoinOrder = false; - const bool Configuration::DefaultValue::replicatedOnly = false; - const bool Configuration::DefaultValue::collocated = false; - const bool Configuration::DefaultValue::lazy = false; - const bool Configuration::DefaultValue::updateOnServer = false; + const bool Configuration::DefaultValue::distributedJoins = false; + const bool Configuration::DefaultValue::enforceJoinOrder = false; + const bool Configuration::DefaultValue::replicatedOnly = false; + const bool Configuration::DefaultValue::collocated = false; + const bool Configuration::DefaultValue::lazy = false; + const bool Configuration::DefaultValue::skipReducerOnUpdate = false; const ProtocolVersion& Configuration::DefaultValue::protocolVersion = ProtocolVersion::GetCurrent(); diff --git a/modules/platforms/cpp/odbc/src/connection.cpp b/modules/platforms/cpp/odbc/src/connection.cpp index ecff413041c56..8f4bf14b94312 100644 --- a/modules/platforms/cpp/odbc/src/connection.cpp +++ b/modules/platforms/cpp/odbc/src/connection.cpp @@ -417,7 +417,7 @@ namespace ignite bool replicatedOnly = false; bool collocated = false; bool lazy = false; - bool updateOnServer = false; + bool skipReducerOnUpdate = false; ProtocolVersion protocolVersion; try @@ -428,7 +428,7 @@ namespace ignite replicatedOnly = config.IsReplicatedOnly(); collocated = config.IsCollocated(); lazy = config.IsLazy(); - updateOnServer = config.IsUpdateOnServer(); + skipReducerOnUpdate = config.IsSkipReducerOnUpdate(); } catch (const IgniteError& err) { @@ -446,7 +446,7 @@ namespace ignite } HandshakeRequest req(protocolVersion, distributedJoins, enforceJoinOrder, replicatedOnly, collocated, lazy, - updateOnServer); + skipReducerOnUpdate); HandshakeResponse rsp; try diff --git a/modules/platforms/cpp/odbc/src/dsn_config.cpp b/modules/platforms/cpp/odbc/src/dsn_config.cpp index 96812a1350ea0..536f6797319b1 100644 --- a/modules/platforms/cpp/odbc/src/dsn_config.cpp +++ b/modules/platforms/cpp/odbc/src/dsn_config.cpp @@ -108,7 +108,8 @@ namespace ignite bool lazy = ReadDsnBool(dsn, Configuration::Key::lazy, config.IsLazy()); - bool updateOnServer = ReadDsnBool(dsn, Configuration::Key::updateOnServer, config.IsUpdateOnServer()); + bool skipReducerOnUpdate = + ReadDsnBool(dsn, Configuration::Key::skipReducerOnUpdate, config.IsSkipReducerOnUpdate()); std::string version = ReadDsnString(dsn, Configuration::Key::protocolVersion, config.GetProtocolVersion().ToString().c_str()); @@ -127,7 +128,7 @@ namespace ignite config.SetReplicatedOnly(replicatedOnly); config.SetCollocated(collocated); config.SetLazy(lazy); - config.SetUpdateOnServer(updateOnServer); + config.SetSkipReducerOnUpdate(skipReducerOnUpdate); config.SetProtocolVersion(version); config.SetPageSize(pageSize); } diff --git a/modules/platforms/cpp/odbc/src/message.cpp b/modules/platforms/cpp/odbc/src/message.cpp index fbdfc376bce5b..4767c74b2424f 100644 --- a/modules/platforms/cpp/odbc/src/message.cpp +++ b/modules/platforms/cpp/odbc/src/message.cpp @@ -23,14 +23,14 @@ namespace ignite namespace odbc { HandshakeRequest::HandshakeRequest(const ProtocolVersion& version, bool distributedJoins, - bool enforceJoinOrder, bool replicatedOnly, bool collocated, bool lazy, bool updateOnServer): + bool enforceJoinOrder, bool replicatedOnly, bool collocated, bool lazy, bool skipReducerOnUpdate): version(version), distributedJoins(distributedJoins), enforceJoinOrder(enforceJoinOrder), replicatedOnly(replicatedOnly), collocated(collocated), lazy(lazy), - updateOnServer(updateOnServer) + skipReducerOnUpdate(skipReducerOnUpdate) { // No-op. } @@ -59,7 +59,7 @@ namespace ignite writer.WriteBool(lazy); if (version >= ProtocolVersion::VERSION_2_3_0) - writer.WriteBool(updateOnServer); + writer.WriteBool(skipReducerOnUpdate); } QueryExecuteRequest::QueryExecuteRequest(const std::string& schema, const std::string& sql, From 863899c330d9eb384182ca457c7de84bb14eac8d Mon Sep 17 00:00:00 2001 From: devozerov Date: Fri, 13 Oct 2017 10:42:25 +0300 Subject: [PATCH 40/40] Cosmetics. --- .../jdbc/suite/IgniteJdbcDriverTestSuite.java | 8 +- ...plexDmlDdlSkipReducerOnUpdateSelfTest.java | 2 +- ...tStatementSkipReducerOnUpdateSelfTest.java | 2 +- ...eStatementSkipReducerOnUpdateSelfTest.java | 2 +- ...eStatementSkipReducerOnUpdateSelfTest.java | 2 +- .../cache/query/SqlFieldsQueryEx.java | 6 +- .../odbc/odbc/OdbcConnectionContext.java | 2 +- .../processors/query/h2/H2DmlPlanKey.java | 2 +- .../processors/query/h2/UpdateResult.java | 4 +- .../query/h2/dml/UpdatePlanBuilder.java | 4 +- .../h2/twostep/DistributedUpdateRun.java | 4 +- .../h2/twostep/msg/GridH2DmlRequest.java | 4 +- .../h2/twostep/msg/GridH2DmlResponse.java | 4 +- ...qlSkipReducerOnUpdateDmlFlagSelfTest.java} | 1564 +++++++++-------- ...iteSqlSkipReducerOnUpdateDmlSelfTest.java} | 12 +- .../IgniteCacheQuerySelfTestSuite.java | 8 +- 16 files changed, 821 insertions(+), 809 deletions(-) rename modules/clients/src/test/java/org/apache/ignite/jdbc/thin/{updateonserver => }/JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.java (96%) rename modules/clients/src/test/java/org/apache/ignite/jdbc/thin/{updateonserver => }/JdbcThinInsertStatementSkipReducerOnUpdateSelfTest.java (96%) rename modules/clients/src/test/java/org/apache/ignite/jdbc/thin/{updateonserver => }/JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.java (96%) rename modules/clients/src/test/java/org/apache/ignite/jdbc/thin/{updateonserver => }/JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest.java (96%) rename modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/{IgniteSqlDistributedDmlFlagSelfTest.java => IgniteSqlSkipReducerOnUpdateDmlFlagSelfTest.java} (95%) rename modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/{IgniteSqlDistributedDmlSelfTest.java => IgniteSqlSkipReducerOnUpdateDmlSelfTest.java} (98%) diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java index c631af41870f4..bec388a0a9bb4 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java @@ -58,10 +58,10 @@ import org.apache.ignite.jdbc.thin.JdbcThinSelectAfterAlterTable; import org.apache.ignite.jdbc.thin.JdbcThinStatementSelfTest; import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSelfTest; -import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest; -import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinInsertStatementSkipReducerOnUpdateSelfTest; -import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinMergeStatementSkipReducerOnUpdateSelfTest; -import org.apache.ignite.jdbc.thin.updateonserver.JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest; +import org.apache.ignite.jdbc.thin.JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest; +import org.apache.ignite.jdbc.thin.JdbcThinInsertStatementSkipReducerOnUpdateSelfTest; +import org.apache.ignite.jdbc.thin.JdbcThinMergeStatementSkipReducerOnUpdateSelfTest; +import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest; /** * JDBC driver test suite. diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.java similarity index 96% rename from modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.java rename to modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.java index 06b0538a93ea6..7ae64792634c3 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinComplexDmlDdlSkipReducerOnUpdateSelfTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.jdbc.thin.updateonserver; +package org.apache.ignite.jdbc.thin; import java.sql.Connection; import java.sql.DriverManager; diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinInsertStatementSkipReducerOnUpdateSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinInsertStatementSkipReducerOnUpdateSelfTest.java similarity index 96% rename from modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinInsertStatementSkipReducerOnUpdateSelfTest.java rename to modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinInsertStatementSkipReducerOnUpdateSelfTest.java index 1cf32f97fccb8..d99639fa1b8d3 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinInsertStatementSkipReducerOnUpdateSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinInsertStatementSkipReducerOnUpdateSelfTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.jdbc.thin.updateonserver; +package org.apache.ignite.jdbc.thin; import java.sql.Connection; import java.sql.DriverManager; diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.java similarity index 96% rename from modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.java rename to modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.java index 2a573353504c4..0832fb711d3a7 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinMergeStatementSkipReducerOnUpdateSelfTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.jdbc.thin.updateonserver; +package org.apache.ignite.jdbc.thin; import java.sql.Connection; import java.sql.DriverManager; diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest.java similarity index 96% rename from modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest.java rename to modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest.java index aef7492a24dc6..475a77f23cc28 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/updateonserver/JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinUpdateStatementSkipReducerOnUpdateSelfTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ignite.jdbc.thin.updateonserver; +package org.apache.ignite.jdbc.thin; import java.sql.Connection; import java.sql.DriverManager; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java index ef8f2783de46d..c5f786ec45a0c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/SqlFieldsQueryEx.java @@ -131,11 +131,11 @@ public Boolean isQuery() { * Only affects DML commands. Ignored when {@link #isLocal()} is {@code true}. * Note that when set to {@code true}, the query may fail in the case of even single node failure. * - * @param updateOnServer Server side update flag. + * @param skipReducerOnUpdate Server side update flag. * @return {@code this} For chaining. */ - public SqlFieldsQuery setSkipReducerOnUpdate(boolean updateOnServer) { - this.skipReducerOnUpdate = updateOnServer; + public SqlFieldsQuery setSkipReducerOnUpdate(boolean skipReducerOnUpdate) { + this.skipReducerOnUpdate = skipReducerOnUpdate; return this; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java index a42e7801165ec..88a2e0ff1cd34 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java @@ -37,7 +37,7 @@ public class OdbcConnectionContext implements ClientListenerConnectionContext { /** Version 2.1.5: added "lazy" flag. */ public static final ClientListenerProtocolVersion VER_2_1_5 = ClientListenerProtocolVersion.create(2, 1, 5); - /** Version 2.3.0: added "updateOnServer" flag. */ + /** Version 2.3.0: added "skipReducerOnUpdate" flag. */ public static final ClientListenerProtocolVersion VER_2_3_0 = ClientListenerProtocolVersion.create(2, 3, 0); /** Current version. */ diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java index cbee2756bcd90..455b5e5a3b94d 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2DmlPlanKey.java @@ -45,7 +45,7 @@ public H2DmlPlanKey(String schemaName, String sql, boolean loc, SqlFieldsQuery f this.schemaName = schemaName; this.sql = sql; - if (loc || !UpdatePlanBuilder.isUpdateOnServerQuery(fieldsQry)) + if (loc || !UpdatePlanBuilder.isSkipReducerOnUpdateQuery(fieldsQry)) this.flags = 0; // flags only relevant for server side updates. else { this.flags = (byte)(1 + diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java index 8c2fc7e569615..c9cfc179fb7aa 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java @@ -20,7 +20,9 @@ import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; -/** Update result - modifications count and keys to re-run query with, if needed. */ +/** + * Update result - modifications count and keys to re-run query with, if needed. + */ public final class UpdateResult { /** Result to return for operations that affected 1 item - mostly to be used for fast updates and deletes. */ final static UpdateResult ONE = new UpdateResult(1, X.EMPTY_OBJECT_ARRAY); diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java index 11e4e281f956e..c84526692250e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java @@ -550,7 +550,7 @@ private static UpdatePlan.DistributedPlanInfo checkPlanCanBeDistributed(IgniteH2 Connection conn, SqlFieldsQuery fieldsQry, boolean loc, String selectQry, String cacheName) throws IgniteCheckedException { - if (loc || !isUpdateOnServerQuery(fieldsQry)) + if (loc || !isSkipReducerOnUpdateQuery(fieldsQry)) return null; assert conn != null; @@ -585,7 +585,7 @@ private static UpdatePlan.DistributedPlanInfo checkPlanCanBeDistributed(IgniteH2 * @param qry Query. * @return {@code true} if update can be distributed. */ - public static boolean isUpdateOnServerQuery(SqlFieldsQuery qry) { + public static boolean isSkipReducerOnUpdateQuery(SqlFieldsQuery qry) { return qry != null && !qry.isLocal() && qry instanceof SqlFieldsQueryEx && ((SqlFieldsQueryEx)qry).isSkipReducerOnUpdate(); } diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java index ffd75adc16af3..a783b8a26b13c 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/DistributedUpdateRun.java @@ -29,7 +29,9 @@ import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.typedef.F; -/** Context for DML operation on reducer node. */ +/** + * Context for DML operation on reducer node. + */ class DistributedUpdateRun { /** Expected number of responses. */ private final int nodeCount; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java index a0b96422d37d5..e40bc2da87ab6 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlRequest.java @@ -38,7 +38,9 @@ import static org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery.EMPTY_PARAMS; -/** Request for DML operation on remote node. */ +/** + * Request for DML operation on remote node. + */ public class GridH2DmlRequest implements Message, GridCacheQueryMarshallable { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java index 04502cd403020..808ff9eb160aa 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2DmlResponse.java @@ -32,7 +32,9 @@ import org.apache.ignite.plugin.extensions.communication.MessageReader; import org.apache.ignite.plugin.extensions.communication.MessageWriter; -/** Response to remote DML request. */ +/** + * Response to remote DML request. + */ public class GridH2DmlResponse implements Message, GridCacheQueryMarshallable { /** */ private static final long serialVersionUID = 0L; diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSkipReducerOnUpdateDmlFlagSelfTest.java similarity index 95% rename from modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java rename to modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSkipReducerOnUpdateDmlFlagSelfTest.java index 56d044e54fc63..e5efc06ef09e0 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlFlagSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSkipReducerOnUpdateDmlFlagSelfTest.java @@ -1,781 +1,783 @@ -package org.apache.ignite.internal.processors.query; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import javax.cache.Cache; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.cache.CacheMode; -import org.apache.ignite.cache.QueryEntity; -import org.apache.ignite.cache.query.annotations.QuerySqlField; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; -import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; -import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; - -/** Tests for {@link SqlFieldsQueryEx#updateOnServer} flag. */ -public class IgniteSqlDistributedDmlFlagSelfTest extends GridCommonAbstractTest { - /** IP finder. */ - private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); - - /** */ - private static int NODE_COUNT = 4; - - /** */ - private static String NODE_CLIENT = "client"; - - /** */ - private static String CACHE_ACCOUNT = "acc"; - - /** */ - private static String CACHE_REPORT = "rep"; - - /** */ - private static String CACHE_STOCK = "stock"; - - /** */ - private static String CACHE_TRADE = "trade"; - - /** */ - private static String CACHE_LIST = "list"; - - /** */ - private static IgniteEx client; - - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { - IgniteConfiguration c = super.getConfiguration(gridName); - - TcpDiscoverySpi disco = new TcpDiscoverySpi(); - - disco.setIpFinder(IP_FINDER); - - c.setDiscoverySpi(disco); - - List ccfgs = new ArrayList<>(); - - ccfgs.add(buildCacheConfiguration(CACHE_ACCOUNT)); - ccfgs.add(buildCacheConfiguration(CACHE_STOCK)); - ccfgs.add(buildCacheConfiguration(CACHE_TRADE)); - ccfgs.add(buildCacheConfiguration(CACHE_REPORT)); - ccfgs.add(buildCacheConfiguration(CACHE_LIST)); - - c.setCacheConfiguration(ccfgs.toArray(new CacheConfiguration[ccfgs.size()])); - - if (gridName.equals(NODE_CLIENT)) - c.setClientMode(true); - - return c; - } - - /** - * Creates a cache configuration. - * - * @param name Name of the cache. - * @return Cache configuration. - */ - private CacheConfiguration buildCacheConfiguration(String name) { - if (name.equals(CACHE_ACCOUNT)) { - CacheConfiguration ccfg = new CacheConfiguration(CACHE_ACCOUNT); - - ccfg.setCacheMode(CacheMode.PARTITIONED); - - QueryEntity entity = new QueryEntity(Integer.class, Account.class); - - ccfg.setQueryEntities(Collections.singletonList(entity)); - - return ccfg; - } - if (name.equals(CACHE_STOCK)) { - CacheConfiguration ccfg = new CacheConfiguration(CACHE_STOCK); - - ccfg.setCacheMode(CacheMode.REPLICATED); - - QueryEntity entity = new QueryEntity(Integer.class, Stock.class); - - ccfg.setQueryEntities(Collections.singletonList(entity)); - - return ccfg; - } - if (name.equals(CACHE_TRADE)) { - CacheConfiguration ccfg = new CacheConfiguration(CACHE_TRADE); - - ccfg.setCacheMode(CacheMode.PARTITIONED); - - QueryEntity entity = new QueryEntity(Integer.class, Trade.class); - - ccfg.setQueryEntities(Collections.singletonList(entity)); - - return ccfg; - } - if (name.equals(CACHE_REPORT)) { - CacheConfiguration ccfg = new CacheConfiguration(CACHE_REPORT); - - ccfg.setCacheMode(CacheMode.PARTITIONED); - - QueryEntity entity = new QueryEntity(Integer.class, Report.class); - - ccfg.setQueryEntities(Collections.singletonList(entity)); - - return ccfg; - } - if (name.equals(CACHE_LIST)) { - CacheConfiguration ccfg = new CacheConfiguration(CACHE_LIST); - - ccfg.setCacheMode(CacheMode.PARTITIONED); - - QueryEntity entity = new QueryEntity(Integer.class, String.class); - - ccfg.setQueryEntities(Collections.singletonList(entity)); - - return ccfg; - } - - assert false; - - return null; - } - - /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - super.beforeTestsStarted(); - - startGrids(NODE_COUNT); - - client = (IgniteEx)startGrid(NODE_CLIENT); - - awaitPartitionMapExchange(); - } - - /** {@inheritDoc} */ - @Override protected void afterTestsStopped() throws Exception { - - super.afterTestsStopped(); - - stopAllGrids(); - } - - /** {@inheritDoc} */ - @Override protected void afterTest() throws Exception { - super.afterTest(); - - awaitPartitionMapExchange(); - - client.cache(CACHE_ACCOUNT).clear(); - client.cache(CACHE_STOCK).clear(); - client.cache(CACHE_TRADE).clear(); - client.cache(CACHE_REPORT).clear(); - client.cache(CACHE_LIST).clear(); - } - - /** - * - * @throws Exception If failed. - */ - public void testUpdate() throws Exception { - Map accounts = getAccounts(100, 1, 100); - - String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE depo > 0"; - - checkUpdate(client.cache(CACHE_ACCOUNT), accounts, new SqlFieldsQueryEx(text, false).setArgs(10)); - } - - /** - * - * @throws Exception If failed. - */ - public void testUpdateFastKey() throws Exception { - Map accounts = getAccounts(100, 1, 100); - - String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE _key = ?"; - - checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQueryEx(text, false).setArgs(10, 1)); - } - - /** - * - * @throws Exception If failed. - */ - public void testUpdateLimit() throws Exception { - Map accounts = getAccounts(100, 1, 100); - - String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE sn >= ? AND sn < ? LIMIT ?"; - - checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQueryEx(text, false).setArgs(10, 0, 10, 10)); - } - - /** - * - * @throws Exception If failed. - */ - public void testUpdateWhereSubquery() throws Exception { - Map accounts = getAccounts(100, 1, -100); - - Map trades = getTrades(100, 2); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - - String text = "UPDATE \"trade\".Trade t SET qty = ? " + - "WHERE accountId IN (SELECT p._key FROM \"acc\".Account p WHERE depo < ?)"; - - checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQueryEx(text, false).setArgs(0, 0)); - } - - /** - * - * @throws Exception If failed. - */ - public void testUpdateSetSubquery() throws Exception { - Map accounts = getAccounts(100, 1, 1000); - Map trades = getTrades(100, 2); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - - String text = "UPDATE \"trade\".Trade t SET qty = " + - "(SELECT a.depo/t.price FROM \"acc\".Account a WHERE t.accountId = a._key)"; - - checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQueryEx(text, false)); - } - - /** - * - * @throws Exception If failed. - */ - public void testUpdateSetTableSubquery() throws Exception { - Map accounts = getAccounts(100, 1, 1000); - Map trades = getTrades(100, 2); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - - String text = "UPDATE \"trade\".Trade t SET (qty) = " + - "(SELECT a.depo/t.price FROM \"acc\".Account a WHERE t.accountId = a._key)"; - - checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQueryEx(text, false)); - } - - /** - * - * @throws Exception If failed. - */ - public void testInsertValues() throws Exception { - String text = "INSERT INTO \"acc\".Account (_key, name, sn, depo)" + - " VALUES (?, ?, ?, ?), (?, ?, ?, ?)"; - - checkUpdate(client.cache(CACHE_ACCOUNT), null, - new SqlFieldsQueryEx(text, false).setArgs(1, "John Marry", 11111, 100, 2, "Marry John", 11112, 200)); - } - - /** - * - * @throws Exception If failed. - */ - public void testInsertFromSelect() throws Exception { - Map accounts = getAccounts(100, 1, 1000); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - - String text = "INSERT INTO \"trade\".Trade (_key, accountId, stockId, qty, price) " + - "SELECT a._key, a._key, ?, a.depo/?, ? FROM \"acc\".Account a"; - - checkUpdate(client.cache(CACHE_TRADE), null, - new SqlFieldsQueryEx(text, false).setArgs(1, 10, 10)); - } - - /** - * - * @throws Exception If failed. - */ - public void testInsertFromSelectOrderBy() throws Exception { - Map accounts = getAccounts(100, 1, 1000); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - - String text = "INSERT INTO \"trade\".Trade (_key, accountId, stockId, qty, price) " + - "SELECT a._key, a._key, ?, a.depo/?, ? FROM \"acc\".Account a " + - "ORDER BY a.sn DESC"; - - checkUpdate(client.cache(CACHE_TRADE), null, - new SqlFieldsQueryEx(text, false).setArgs(1, 10, 10)); - } - - /** - * - * @throws Exception If failed. - */ - public void testInsertFromSelectUnion() throws Exception { - Map accounts = getAccounts(20, 1, 1000); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - - String text = "INSERT INTO \"trade\".Trade (_key, accountId, stockId, qty, price) " + - "SELECT a._key, a._key, 0, a.depo, 1 FROM \"acc\".Account a " + - "UNION " + - "SELECT 101 + a2._key, a2._key, 1, a2.depo, 1 FROM \"acc\".Account a2"; - - checkUpdate(client.cache(CACHE_TRADE), null, - new SqlFieldsQueryEx(text, false)); - } - - /** - * - * @throws Exception If failed. - */ - public void testInsertFromSelectGroupBy() throws Exception { - Map accounts = getAccounts(100, 1, 1000); - Map trades = getTrades(100, 2); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - client.cache(CACHE_TRADE).putAll(trades); - - String text = "INSERT INTO \"rep\".Report (_key, accountId, spends, count) " + - "SELECT accountId, accountId, SUM(qty * price), COUNT(*) " + - "FROM \"trade\".Trade " + - "GROUP BY accountId"; - - checkUpdate(client.cache(CACHE_REPORT), null, - new SqlFieldsQueryEx(text, false)); - } - - /** - * - * @throws Exception If failed. - */ - public void testInsertFromSelectDistinct() throws Exception { - Map accounts = getAccounts(100, 2, 100); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - - String text = "INSERT INTO \"list\".String (_key, _val) " + - "SELECT DISTINCT sn, name FROM \"acc\".Account "; - - checkUpdate(client.cache(CACHE_LIST), null, - new SqlFieldsQueryEx(text, false)); - } - - /** - * - * @throws Exception If failed. - */ - public void testInsertFromSelectJoin() throws Exception { - Map accounts = getAccounts(100, 1, 100); - Map stocks = getStocks(5); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - client.cache(CACHE_STOCK).putAll(stocks); - - String text = "INSERT INTO \"trade\".Trade(_key, accountId, stockId, qty, price) " + - "SELECT 5*a._key + s._key, a._key, s._key, ?, a.depo/? " + - "FROM \"acc\".Account a JOIN \"stock\".Stock s ON 1=1"; - - checkUpdate(client.cache(CACHE_TRADE), null, - new SqlFieldsQueryEx(text, false).setArgs(10, 10)); - } - - /** - * - * @throws Exception If failed. - */ - public void testDelete() throws Exception { - Map accounts = getAccounts(100, 1, 100); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - - String text = "DELETE FROM \"acc\".Account WHERE sn > ?"; - - checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQueryEx(text, false).setArgs(10)); - } - - /** - * - * @throws Exception If failed. - */ - public void testDeleteTop() throws Exception { - Map accounts = getAccounts(100, 1, 100); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - - String text = "DELETE TOP ? FROM \"acc\".Account WHERE sn < ?"; - - checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQueryEx(text, false).setArgs(10, 10)); - } - - /** - * - * @throws Exception If failed. - */ - public void testDeleteWhereSubquery() throws Exception { - Map accounts = getAccounts(20, 1, 100); - Map trades = getTrades(10, 2); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - client.cache(CACHE_TRADE).putAll(trades); - - String text = "DELETE FROM \"acc\".Account " + - "WHERE _key IN (SELECT t.accountId FROM \"trade\".Trade t)"; - - checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQueryEx(text, false)); - } - - /** - * - * @throws Exception If failed. - */ - public void testMergeValues() throws Exception { - Map accounts = getAccounts(1, 1, 100); - - String text = "MERGE INTO \"acc\".Account (_key, name, sn, depo)" + - " VALUES (?, ?, ?, ?), (?, ?, ?, ?)"; - - checkUpdate(client.cache(CACHE_ACCOUNT), accounts, - new SqlFieldsQueryEx(text, false).setArgs(0, "John Marry", 11111, 100, 1, "Marry John", 11112, 200)); - } - - /** - * - * @throws Exception If failed. - */ - public void testMergeFromSelectJoin() throws Exception { - Map accounts = getAccounts(100, 1, 100); - Map stocks = getStocks(5); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - client.cache(CACHE_STOCK).putAll(stocks); - - Map trades = new HashMap<>(); - - trades.put(5, new Trade(1, 1, 1, 1)); - - String text = "MERGE INTO \"trade\".Trade(_key, accountId, stockId, qty, price) " + - "SELECT 5*a._key + s._key, a._key, s._key, ?, a.depo/? " + - "FROM \"acc\".Account a JOIN \"stock\".Stock s ON 1=1"; - - checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQueryEx(text, false).setArgs(10, 10)); - } - - /** - * - * @throws Exception If failed. - */ - public void testMergeFromSelectOrderBy() throws Exception { - Map accounts = getAccounts(100, 1, 1000); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - - Map trades = new HashMap<>(); - - trades.put(5, new Trade(1, 1, 1, 1)); - - String text = "MERGE INTO \"trade\".Trade (_key, accountId, stockId, qty, price) " + - "SELECT a._key, a._key, ?, a.depo/?, ? FROM \"acc\".Account a " + - "ORDER BY a.sn DESC"; - - checkUpdate(client.cache(CACHE_TRADE), trades, - new SqlFieldsQueryEx(text, false).setArgs(1, 10, 10)); - } - - /** - * - * @throws Exception If failed. - */ - public void testMergeFromSelectGroupBy() throws Exception { - Map accounts = getAccounts(100, 1, 1000); - Map trades = getTrades(100, 2); - - client.cache(CACHE_ACCOUNT).putAll(accounts); - client.cache(CACHE_TRADE).putAll(trades); - - Map reports = new HashMap<>(); - - reports.put(5, new Report(5, 1, 1)); - - String text = "MERGE INTO \"rep\".Report (_key, accountId, spends, count) " + - "SELECT accountId, accountId, SUM(qty * price), COUNT(*) " + - "FROM \"trade\".Trade " + - "GROUP BY accountId"; - - checkUpdate(client.cache(CACHE_REPORT), reports, - new SqlFieldsQueryEx(text, false)); - } - - /** - * Constructs multiple Account objects. - * - * @param num Number of accounts. - * @param numCopy Number of copies. - * @param depo Deposit amount. - * @return Map of accounts. - */ - private Map getAccounts(int num, int numCopy, int depo) { - Map res = new HashMap<>(); - - int count = 0; - - for (int i = 0; i < num; ++i) { - String name = "John doe #" + i; - - for (int j = 0; j < numCopy; ++j) - res.put(count++, new Account(name, i, depo)); - } - - return res; - } - - /** - * Constructs multiple Stock objects. - * - * @param num Number of stocks. - * @return Map of Stock objects. - */ - private Map getStocks(int num) { - Map res = new HashMap<>(); - - for (int i = 0; i < num; ++i) - res.put(i, new Stock("T" + i, "Stock #" + i)); - - return res; - } - - /** - * Constructs multiple Trade objects. - * - * @param numAccounts Number of accounts. - * @param numStocks Number of stocks. - * @return Map of Trade objects. - */ - private Map getTrades(int numAccounts, int numStocks) { - Map res = new HashMap<>(); - - int count = 0; - - for (int i = 0; i < numAccounts; ++i) { - for (int j = 0; j < numStocks; ++j) { - res.put(count++, new Trade(i, j, 100, 100)); - } - } - - return res; - } - - /** - * Executes provided sql update with updateOnServer flag on and off and checks results are the same. - * - * @param cache Cache. - * @param initial Initial content of the cache. - * @param qry Query to execute. - * @param Key type. - * @param Value type. - */ - private void checkUpdate(IgniteCache cache, Map initial, SqlFieldsQueryEx qry) { - cache.clear(); - - if (!F.isEmpty(initial)) - cache.putAll(initial); - - List> updRes = cache.query(qry.setSkipReducerOnUpdate(true)).getAll(); - - Map result = new HashMap<>(cache.size()); - - for (Cache.Entry e : cache) - result.put(e.getKey(), e.getValue()); - - cache.clear(); - - if (!F.isEmpty(initial)) - cache.putAll(initial); - - List> updRes2 = cache.query(qry.setSkipReducerOnUpdate(false)).getAll(); - - assertTrue(((Number)updRes.get(0).get(0)).intValue() > 0); - - assertEquals(((Number)updRes.get(0).get(0)).intValue(), ((Number)updRes2.get(0).get(0)).intValue()); - - assertEquals(result.size(), cache.size()); - - for (Cache.Entry e : cache) - assertEquals(e.getValue(), result.get(e.getKey())); - } - - /** */ - public class Account { - /** */ - @QuerySqlField - String name; - - /** */ - @QuerySqlField - int sn; - - /** */ - @QuerySqlField - int depo; - - /** - * Constructor. - * - * @param name Name. - * @param sn ID. - * @param depo Deposit amount. - */ - Account(String name, int sn, int depo) { - this.name = name; - this.sn = sn; - this.depo = depo; - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return (name == null ? 0 : name.hashCode()) ^ sn ^ depo; - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object obj) { - if (obj == null) - return false; - - if (!obj.getClass().equals(Account.class)) - return false; - - Account other = (Account)obj; - - return F.eq(name, other.name) && sn == other.sn && depo == other.depo; - } - } - - /** */ - public class Stock { - /** */ - @QuerySqlField - String ticker; - - /** */ - @QuerySqlField - String name; - - /** - * Constructor. - * - * @param ticker Short name. - * @param name Name. - */ - Stock(String ticker, String name) { - this.ticker = ticker; - this.name = name; - } - } - - /** */ - public class Trade { - /** */ - @QuerySqlField - int accountId; - - /** */ - @QuerySqlField - int stockId; - - /** */ - @QuerySqlField - int qty; - - /** */ - @QuerySqlField - int price; - - /** - * Constructor. - * - * @param accountId Account id. - * @param stockId Stock id. - * @param qty Quantity. - * @param price Price. - */ - Trade(int accountId, int stockId, int qty, int price) { - this.accountId = accountId; - this.stockId = stockId; - this.qty = qty; - this.price = price; - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return accountId ^ stockId ^ qty ^ price; - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object obj) { - if (obj == null) - return false; - - if (!obj.getClass().equals(Trade.class)) - return false; - - Trade other = (Trade)obj; - - return accountId == other.accountId && stockId == other.stockId && - qty == other.qty && price == other.price; - } - - } - - /** */ - public class Report { - /** */ - @QuerySqlField - int accountId; - - /** */ - @QuerySqlField - int spends; - - /** */ - @QuerySqlField - int count; - - /** - * Constructor. - * - * @param accountId Account id. - * @param spends Spends. - * @param count Count. - */ - Report(int accountId, int spends, int count) { - this.accountId = accountId; - this.spends = spends; - this.count = count; - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return accountId ^ spends ^ count; - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object obj) { - if (obj == null) - return false; - - if (!obj.getClass().equals(Report.class)) - return false; - - Report other = (Report)obj; - - return accountId == other.accountId && spends == other.spends && - count == other.count; - } - } -} +package org.apache.ignite.internal.processors.query; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.cache.Cache; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.query.annotations.QuerySqlField; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.query.SqlFieldsQueryEx; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * Tests for {@link SqlFieldsQueryEx#skipReducerOnUpdate} flag. + */ +public class IgniteSqlSkipReducerOnUpdateDmlFlagSelfTest extends GridCommonAbstractTest { + /** IP finder. */ + private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** */ + private static int NODE_COUNT = 4; + + /** */ + private static String NODE_CLIENT = "client"; + + /** */ + private static String CACHE_ACCOUNT = "acc"; + + /** */ + private static String CACHE_REPORT = "rep"; + + /** */ + private static String CACHE_STOCK = "stock"; + + /** */ + private static String CACHE_TRADE = "trade"; + + /** */ + private static String CACHE_LIST = "list"; + + /** */ + private static IgniteEx client; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration c = super.getConfiguration(gridName); + + TcpDiscoverySpi disco = new TcpDiscoverySpi(); + + disco.setIpFinder(IP_FINDER); + + c.setDiscoverySpi(disco); + + List ccfgs = new ArrayList<>(); + + ccfgs.add(buildCacheConfiguration(CACHE_ACCOUNT)); + ccfgs.add(buildCacheConfiguration(CACHE_STOCK)); + ccfgs.add(buildCacheConfiguration(CACHE_TRADE)); + ccfgs.add(buildCacheConfiguration(CACHE_REPORT)); + ccfgs.add(buildCacheConfiguration(CACHE_LIST)); + + c.setCacheConfiguration(ccfgs.toArray(new CacheConfiguration[ccfgs.size()])); + + if (gridName.equals(NODE_CLIENT)) + c.setClientMode(true); + + return c; + } + + /** + * Creates a cache configuration. + * + * @param name Name of the cache. + * @return Cache configuration. + */ + private CacheConfiguration buildCacheConfiguration(String name) { + if (name.equals(CACHE_ACCOUNT)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_ACCOUNT); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + QueryEntity entity = new QueryEntity(Integer.class, Account.class); + + ccfg.setQueryEntities(Collections.singletonList(entity)); + + return ccfg; + } + if (name.equals(CACHE_STOCK)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_STOCK); + + ccfg.setCacheMode(CacheMode.REPLICATED); + + QueryEntity entity = new QueryEntity(Integer.class, Stock.class); + + ccfg.setQueryEntities(Collections.singletonList(entity)); + + return ccfg; + } + if (name.equals(CACHE_TRADE)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_TRADE); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + QueryEntity entity = new QueryEntity(Integer.class, Trade.class); + + ccfg.setQueryEntities(Collections.singletonList(entity)); + + return ccfg; + } + if (name.equals(CACHE_REPORT)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_REPORT); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + QueryEntity entity = new QueryEntity(Integer.class, Report.class); + + ccfg.setQueryEntities(Collections.singletonList(entity)); + + return ccfg; + } + if (name.equals(CACHE_LIST)) { + CacheConfiguration ccfg = new CacheConfiguration(CACHE_LIST); + + ccfg.setCacheMode(CacheMode.PARTITIONED); + + QueryEntity entity = new QueryEntity(Integer.class, String.class); + + ccfg.setQueryEntities(Collections.singletonList(entity)); + + return ccfg; + } + + assert false; + + return null; + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startGrids(NODE_COUNT); + + client = (IgniteEx)startGrid(NODE_CLIENT); + + awaitPartitionMapExchange(); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + + super.afterTestsStopped(); + + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + awaitPartitionMapExchange(); + + client.cache(CACHE_ACCOUNT).clear(); + client.cache(CACHE_STOCK).clear(); + client.cache(CACHE_TRADE).clear(); + client.cache(CACHE_REPORT).clear(); + client.cache(CACHE_LIST).clear(); + } + + /** + * + * @throws Exception If failed. + */ + public void testUpdate() throws Exception { + Map accounts = getAccounts(100, 1, 100); + + String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE depo > 0"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, new SqlFieldsQueryEx(text, false).setArgs(10)); + } + + /** + * + * @throws Exception If failed. + */ + public void testUpdateFastKey() throws Exception { + Map accounts = getAccounts(100, 1, 100); + + String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE _key = ?"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, + new SqlFieldsQueryEx(text, false).setArgs(10, 1)); + } + + /** + * + * @throws Exception If failed. + */ + public void testUpdateLimit() throws Exception { + Map accounts = getAccounts(100, 1, 100); + + String text = "UPDATE \"acc\".Account SET depo = depo - ? WHERE sn >= ? AND sn < ? LIMIT ?"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, + new SqlFieldsQueryEx(text, false).setArgs(10, 0, 10, 10)); + } + + /** + * + * @throws Exception If failed. + */ + public void testUpdateWhereSubquery() throws Exception { + Map accounts = getAccounts(100, 1, -100); + + Map trades = getTrades(100, 2); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "UPDATE \"trade\".Trade t SET qty = ? " + + "WHERE accountId IN (SELECT p._key FROM \"acc\".Account p WHERE depo < ?)"; + + checkUpdate(client.cache(CACHE_TRADE), trades, + new SqlFieldsQueryEx(text, false).setArgs(0, 0)); + } + + /** + * + * @throws Exception If failed. + */ + public void testUpdateSetSubquery() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + Map trades = getTrades(100, 2); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "UPDATE \"trade\".Trade t SET qty = " + + "(SELECT a.depo/t.price FROM \"acc\".Account a WHERE t.accountId = a._key)"; + + checkUpdate(client.cache(CACHE_TRADE), trades, + new SqlFieldsQueryEx(text, false)); + } + + /** + * + * @throws Exception If failed. + */ + public void testUpdateSetTableSubquery() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + Map trades = getTrades(100, 2); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "UPDATE \"trade\".Trade t SET (qty) = " + + "(SELECT a.depo/t.price FROM \"acc\".Account a WHERE t.accountId = a._key)"; + + checkUpdate(client.cache(CACHE_TRADE), trades, + new SqlFieldsQueryEx(text, false)); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertValues() throws Exception { + String text = "INSERT INTO \"acc\".Account (_key, name, sn, depo)" + + " VALUES (?, ?, ?, ?), (?, ?, ?, ?)"; + + checkUpdate(client.cache(CACHE_ACCOUNT), null, + new SqlFieldsQueryEx(text, false).setArgs(1, "John Marry", 11111, 100, 2, "Marry John", 11112, 200)); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertFromSelect() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "INSERT INTO \"trade\".Trade (_key, accountId, stockId, qty, price) " + + "SELECT a._key, a._key, ?, a.depo/?, ? FROM \"acc\".Account a"; + + checkUpdate(client.cache(CACHE_TRADE), null, + new SqlFieldsQueryEx(text, false).setArgs(1, 10, 10)); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertFromSelectOrderBy() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "INSERT INTO \"trade\".Trade (_key, accountId, stockId, qty, price) " + + "SELECT a._key, a._key, ?, a.depo/?, ? FROM \"acc\".Account a " + + "ORDER BY a.sn DESC"; + + checkUpdate(client.cache(CACHE_TRADE), null, + new SqlFieldsQueryEx(text, false).setArgs(1, 10, 10)); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertFromSelectUnion() throws Exception { + Map accounts = getAccounts(20, 1, 1000); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "INSERT INTO \"trade\".Trade (_key, accountId, stockId, qty, price) " + + "SELECT a._key, a._key, 0, a.depo, 1 FROM \"acc\".Account a " + + "UNION " + + "SELECT 101 + a2._key, a2._key, 1, a2.depo, 1 FROM \"acc\".Account a2"; + + checkUpdate(client.cache(CACHE_TRADE), null, + new SqlFieldsQueryEx(text, false)); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertFromSelectGroupBy() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + Map trades = getTrades(100, 2); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + client.cache(CACHE_TRADE).putAll(trades); + + String text = "INSERT INTO \"rep\".Report (_key, accountId, spends, count) " + + "SELECT accountId, accountId, SUM(qty * price), COUNT(*) " + + "FROM \"trade\".Trade " + + "GROUP BY accountId"; + + checkUpdate(client.cache(CACHE_REPORT), null, + new SqlFieldsQueryEx(text, false)); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertFromSelectDistinct() throws Exception { + Map accounts = getAccounts(100, 2, 100); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "INSERT INTO \"list\".String (_key, _val) " + + "SELECT DISTINCT sn, name FROM \"acc\".Account "; + + checkUpdate(client.cache(CACHE_LIST), null, + new SqlFieldsQueryEx(text, false)); + } + + /** + * + * @throws Exception If failed. + */ + public void testInsertFromSelectJoin() throws Exception { + Map accounts = getAccounts(100, 1, 100); + Map stocks = getStocks(5); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + client.cache(CACHE_STOCK).putAll(stocks); + + String text = "INSERT INTO \"trade\".Trade(_key, accountId, stockId, qty, price) " + + "SELECT 5*a._key + s._key, a._key, s._key, ?, a.depo/? " + + "FROM \"acc\".Account a JOIN \"stock\".Stock s ON 1=1"; + + checkUpdate(client.cache(CACHE_TRADE), null, + new SqlFieldsQueryEx(text, false).setArgs(10, 10)); + } + + /** + * + * @throws Exception If failed. + */ + public void testDelete() throws Exception { + Map accounts = getAccounts(100, 1, 100); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "DELETE FROM \"acc\".Account WHERE sn > ?"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, + new SqlFieldsQueryEx(text, false).setArgs(10)); + } + + /** + * + * @throws Exception If failed. + */ + public void testDeleteTop() throws Exception { + Map accounts = getAccounts(100, 1, 100); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + String text = "DELETE TOP ? FROM \"acc\".Account WHERE sn < ?"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, + new SqlFieldsQueryEx(text, false).setArgs(10, 10)); + } + + /** + * + * @throws Exception If failed. + */ + public void testDeleteWhereSubquery() throws Exception { + Map accounts = getAccounts(20, 1, 100); + Map trades = getTrades(10, 2); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + client.cache(CACHE_TRADE).putAll(trades); + + String text = "DELETE FROM \"acc\".Account " + + "WHERE _key IN (SELECT t.accountId FROM \"trade\".Trade t)"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, + new SqlFieldsQueryEx(text, false)); + } + + /** + * + * @throws Exception If failed. + */ + public void testMergeValues() throws Exception { + Map accounts = getAccounts(1, 1, 100); + + String text = "MERGE INTO \"acc\".Account (_key, name, sn, depo)" + + " VALUES (?, ?, ?, ?), (?, ?, ?, ?)"; + + checkUpdate(client.cache(CACHE_ACCOUNT), accounts, + new SqlFieldsQueryEx(text, false).setArgs(0, "John Marry", 11111, 100, 1, "Marry John", 11112, 200)); + } + + /** + * + * @throws Exception If failed. + */ + public void testMergeFromSelectJoin() throws Exception { + Map accounts = getAccounts(100, 1, 100); + Map stocks = getStocks(5); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + client.cache(CACHE_STOCK).putAll(stocks); + + Map trades = new HashMap<>(); + + trades.put(5, new Trade(1, 1, 1, 1)); + + String text = "MERGE INTO \"trade\".Trade(_key, accountId, stockId, qty, price) " + + "SELECT 5*a._key + s._key, a._key, s._key, ?, a.depo/? " + + "FROM \"acc\".Account a JOIN \"stock\".Stock s ON 1=1"; + + checkUpdate(client.cache(CACHE_TRADE), trades, + new SqlFieldsQueryEx(text, false).setArgs(10, 10)); + } + + /** + * + * @throws Exception If failed. + */ + public void testMergeFromSelectOrderBy() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + + Map trades = new HashMap<>(); + + trades.put(5, new Trade(1, 1, 1, 1)); + + String text = "MERGE INTO \"trade\".Trade (_key, accountId, stockId, qty, price) " + + "SELECT a._key, a._key, ?, a.depo/?, ? FROM \"acc\".Account a " + + "ORDER BY a.sn DESC"; + + checkUpdate(client.cache(CACHE_TRADE), trades, + new SqlFieldsQueryEx(text, false).setArgs(1, 10, 10)); + } + + /** + * + * @throws Exception If failed. + */ + public void testMergeFromSelectGroupBy() throws Exception { + Map accounts = getAccounts(100, 1, 1000); + Map trades = getTrades(100, 2); + + client.cache(CACHE_ACCOUNT).putAll(accounts); + client.cache(CACHE_TRADE).putAll(trades); + + Map reports = new HashMap<>(); + + reports.put(5, new Report(5, 1, 1)); + + String text = "MERGE INTO \"rep\".Report (_key, accountId, spends, count) " + + "SELECT accountId, accountId, SUM(qty * price), COUNT(*) " + + "FROM \"trade\".Trade " + + "GROUP BY accountId"; + + checkUpdate(client.cache(CACHE_REPORT), reports, + new SqlFieldsQueryEx(text, false)); + } + + /** + * Constructs multiple Account objects. + * + * @param num Number of accounts. + * @param numCopy Number of copies. + * @param depo Deposit amount. + * @return Map of accounts. + */ + private Map getAccounts(int num, int numCopy, int depo) { + Map res = new HashMap<>(); + + int count = 0; + + for (int i = 0; i < num; ++i) { + String name = "John doe #" + i; + + for (int j = 0; j < numCopy; ++j) + res.put(count++, new Account(name, i, depo)); + } + + return res; + } + + /** + * Constructs multiple Stock objects. + * + * @param num Number of stocks. + * @return Map of Stock objects. + */ + private Map getStocks(int num) { + Map res = new HashMap<>(); + + for (int i = 0; i < num; ++i) + res.put(i, new Stock("T" + i, "Stock #" + i)); + + return res; + } + + /** + * Constructs multiple Trade objects. + * + * @param numAccounts Number of accounts. + * @param numStocks Number of stocks. + * @return Map of Trade objects. + */ + private Map getTrades(int numAccounts, int numStocks) { + Map res = new HashMap<>(); + + int count = 0; + + for (int i = 0; i < numAccounts; ++i) { + for (int j = 0; j < numStocks; ++j) { + res.put(count++, new Trade(i, j, 100, 100)); + } + } + + return res; + } + + /** + * Executes provided sql update with skipReducerOnUpdate flag on and off and checks results are the same. + * + * @param cache Cache. + * @param initial Initial content of the cache. + * @param qry Query to execute. + * @param Key type. + * @param Value type. + */ + private void checkUpdate(IgniteCache cache, Map initial, SqlFieldsQueryEx qry) { + cache.clear(); + + if (!F.isEmpty(initial)) + cache.putAll(initial); + + List> updRes = cache.query(qry.setSkipReducerOnUpdate(true)).getAll(); + + Map result = new HashMap<>(cache.size()); + + for (Cache.Entry e : cache) + result.put(e.getKey(), e.getValue()); + + cache.clear(); + + if (!F.isEmpty(initial)) + cache.putAll(initial); + + List> updRes2 = cache.query(qry.setSkipReducerOnUpdate(false)).getAll(); + + assertTrue(((Number)updRes.get(0).get(0)).intValue() > 0); + + assertEquals(((Number)updRes.get(0).get(0)).intValue(), ((Number)updRes2.get(0).get(0)).intValue()); + + assertEquals(result.size(), cache.size()); + + for (Cache.Entry e : cache) + assertEquals(e.getValue(), result.get(e.getKey())); + } + + /** */ + public class Account { + /** */ + @QuerySqlField + String name; + + /** */ + @QuerySqlField + int sn; + + /** */ + @QuerySqlField + int depo; + + /** + * Constructor. + * + * @param name Name. + * @param sn ID. + * @param depo Deposit amount. + */ + Account(String name, int sn, int depo) { + this.name = name; + this.sn = sn; + this.depo = depo; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return (name == null ? 0 : name.hashCode()) ^ sn ^ depo; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (obj == null) + return false; + + if (!obj.getClass().equals(Account.class)) + return false; + + Account other = (Account)obj; + + return F.eq(name, other.name) && sn == other.sn && depo == other.depo; + } + } + + /** */ + public class Stock { + /** */ + @QuerySqlField + String ticker; + + /** */ + @QuerySqlField + String name; + + /** + * Constructor. + * + * @param ticker Short name. + * @param name Name. + */ + Stock(String ticker, String name) { + this.ticker = ticker; + this.name = name; + } + } + + /** */ + public class Trade { + /** */ + @QuerySqlField + int accountId; + + /** */ + @QuerySqlField + int stockId; + + /** */ + @QuerySqlField + int qty; + + /** */ + @QuerySqlField + int price; + + /** + * Constructor. + * + * @param accountId Account id. + * @param stockId Stock id. + * @param qty Quantity. + * @param price Price. + */ + Trade(int accountId, int stockId, int qty, int price) { + this.accountId = accountId; + this.stockId = stockId; + this.qty = qty; + this.price = price; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return accountId ^ stockId ^ qty ^ price; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (obj == null) + return false; + + if (!obj.getClass().equals(Trade.class)) + return false; + + Trade other = (Trade)obj; + + return accountId == other.accountId && stockId == other.stockId && + qty == other.qty && price == other.price; + } + + } + + /** */ + public class Report { + /** */ + @QuerySqlField + int accountId; + + /** */ + @QuerySqlField + int spends; + + /** */ + @QuerySqlField + int count; + + /** + * Constructor. + * + * @param accountId Account id. + * @param spends Spends. + * @param count Count. + */ + Report(int accountId, int spends, int count) { + this.accountId = accountId; + this.spends = spends; + this.count = count; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return accountId ^ spends ^ count; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (obj == null) + return false; + + if (!obj.getClass().equals(Report.class)) + return false; + + Report other = (Report)obj; + + return accountId == other.accountId && spends == other.spends && + count == other.count; + } + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSkipReducerOnUpdateDmlSelfTest.java similarity index 98% rename from modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java rename to modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSkipReducerOnUpdateDmlSelfTest.java index bf5c645d36604..a2a6bf8ba79fd 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlDistributedDmlSelfTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSkipReducerOnUpdateDmlSelfTest.java @@ -58,9 +58,11 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED; -/** Tests for distributed DML. */ +/** + * Tests for distributed DML. + */ @SuppressWarnings({"unchecked", "ThrowableResultOfMethodCallIgnored"}) -public class IgniteSqlDistributedDmlSelfTest extends GridCommonAbstractTest { +public class IgniteSqlSkipReducerOnUpdateDmlSelfTest extends GridCommonAbstractTest { /** IP finder. */ private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); @@ -127,7 +129,7 @@ private CacheConfiguration buildCacheConfiguration(String name) { ccfg.setQueryEntities(Collections.singletonList(entity)); - ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); + ccfg.setSqlFunctionClasses(IgniteSqlSkipReducerOnUpdateDmlSelfTest.class); return ccfg; } @@ -142,7 +144,7 @@ private CacheConfiguration buildCacheConfiguration(String name) { ccfg.setKeyConfiguration(new CacheKeyConfiguration(PersonKey.class)); - ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); + ccfg.setSqlFunctionClasses(IgniteSqlSkipReducerOnUpdateDmlSelfTest.class); return ccfg; } @@ -155,7 +157,7 @@ private CacheConfiguration buildCacheConfiguration(String name) { ccfg.setQueryEntities(Collections.singletonList(entity)); - ccfg.setSqlFunctionClasses(IgniteSqlDistributedDmlSelfTest.class); + ccfg.setSqlFunctionClasses(IgniteSqlSkipReducerOnUpdateDmlSelfTest.class); return ccfg; } diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java index d6083cf5fc666..83b4689678f5b 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java @@ -123,11 +123,11 @@ import org.apache.ignite.internal.processors.cache.query.IndexingSpiQuerySelfTest; import org.apache.ignite.internal.processors.cache.query.IndexingSpiQueryTxSelfTest; import org.apache.ignite.internal.processors.client.ClientConnectorConfigurationValidationSelfTest; -import org.apache.ignite.internal.processors.query.IgniteSqlDistributedDmlFlagSelfTest; +import org.apache.ignite.internal.processors.query.IgniteSqlSkipReducerOnUpdateDmlFlagSelfTest; import org.apache.ignite.internal.processors.query.IgniteSqlParameterizedQueryTest; import org.apache.ignite.internal.processors.query.h2.IgniteSqlBigIntegerKeyTest; import org.apache.ignite.internal.processors.query.IgniteQueryDedicatedPoolTest; -import org.apache.ignite.internal.processors.query.IgniteSqlDistributedDmlSelfTest; +import org.apache.ignite.internal.processors.query.IgniteSqlSkipReducerOnUpdateDmlSelfTest; import org.apache.ignite.internal.processors.query.IgniteSqlEntryCacheModeAgnosticTest; import org.apache.ignite.internal.processors.query.IgniteSqlKeyValueFieldsTest; import org.apache.ignite.internal.processors.query.IgniteSqlNotNullConstraintTest; @@ -245,8 +245,8 @@ public static TestSuite suite() throws Exception { suite.addTestSuite(IgniteCacheInsertSqlQuerySelfTest.class); suite.addTestSuite(IgniteCacheUpdateSqlQuerySelfTest.class); suite.addTestSuite(IgniteCacheDeleteSqlQuerySelfTest.class); - suite.addTestSuite(IgniteSqlDistributedDmlSelfTest.class); - suite.addTestSuite(IgniteSqlDistributedDmlFlagSelfTest.class); + suite.addTestSuite(IgniteSqlSkipReducerOnUpdateDmlSelfTest.class); + suite.addTestSuite(IgniteSqlSkipReducerOnUpdateDmlFlagSelfTest.class); suite.addTestSuite(IgniteBinaryObjectQueryArgumentsTest.class); suite.addTestSuite(IgniteBinaryObjectLocalQueryArgumentsTest.class);